02:53:50.151 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:53:50.162 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskL3tT2Z/test-tmp
02:53:50.165 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:53:50.169 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskL3tT2Z/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:53:50.385 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:53:50.546 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:53:50.547 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:53:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:50.552385 5047 mini_oidc.cc:152] Starting JWKS server
02:53:50.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:50.556797 5047 webserver.cc:460] Webserver started at http://127.0.0.1:44969/ using document root <none> and password file <none>
02:53:50.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:50.557422 5047 mini_oidc.cc:189] Starting OIDC Discovery server
02:53:50.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:50.559377 5047 webserver.cc:460] Webserver started at http://127.0.0.1:33561/ using document root <none> and password file <none>
02:53:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:53:50Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:53:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:53:50Z Disabled control of system clock
02:53:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:50.604074 5047 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/wal
02:53:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/data
02:53:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:53:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:53:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:53:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/logs
02:53:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/data/info.pb
02:53:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:53:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:53:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:53:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:53:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:53:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:53:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:53:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:53:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.254:41617
02:53:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.254
02:53:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:53:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:53:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:53:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44969/jwks.json with env {}
02:53:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.889214 5060 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:53:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.889796 5060 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:53:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.890314 5060 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:53:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.928416 5060 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:53:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.928699 5060 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:53:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.928900 5060 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:53:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.929072 5060 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:53:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.929286 5060 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:53:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.929447 5060 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44969/jwks.json
02:53:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:50.963294 5060 master_runner.cc:386] Master server non-default flags:
02:53:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:53:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/data
02:53:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/wal
02:53:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.254:41617
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44969/jwks.json
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/data/info.pb
02:53:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.254
02:53:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:53:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:53:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/logs
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:53:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:53:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:53:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:53:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:50.964560 5060 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:53:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:50.966675 5060 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:53:50.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.983901 5069 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:50.984422 5067 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:51.025336 5066 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:51.025624 5060 server_base.cc:1034] running on GCE node
02:53:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.069113 5060 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:53:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.071753 5060 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:53:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.073101 5060 hybrid_clock.cc:648] HybridClock initialized: now 1790650432073069 us; error 50 us; skew 500 ppm
02:53:52.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.073917 5060 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:53:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.080188 5060 webserver.cc:460] Webserver started at http://127.4.237.254:46647/ using document root <none> and password file <none>
02:53:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.081059 5060 fs_manager.cc:359] Metadata directory not provided
02:53:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.081236 5060 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:53:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.081597 5060 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:53:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.086295 5060 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/data/instance:
02:53:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7"
02:53:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:52 on dist-test-slave-sksk"
02:53:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.087529 5060 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/wal/instance:
02:53:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7"
02:53:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:52 on dist-test-slave-sksk"
02:53:52.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.094722 5060 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
02:53:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.100064 5076 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:53:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.101099 5060 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
02:53:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.101420 5060 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/wal
02:53:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7"
02:53:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:52 on dist-test-slave-sksk"
02:53:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.101821 5060 fs_report.cc:383] FS layout report
02:53:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:53:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/wal
02:53:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/wal
02:53:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/data/data
02:53:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:53:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:53:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:53:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:53:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:53:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:53:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:53:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:53:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:53:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:53:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:53:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.140229 5060 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:53:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.141832 5060 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:53:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.142292 5060 kserver.cc:163] Server-wide thread pool size limit: 3276
02:53:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.210230 5060 rpc_server.cc:307] RPC server started. Bound to: 127.4.237.254:41617
02:53:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.210338 5127 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.237.254:41617 every 8 connection(s)
02:53:52.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.212971 5060 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/data/info.pb
02:53:52.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.218146 5128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:53:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.222887 5047 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5060
02:53:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.223465 5047 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-0/wal/instance
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.228569 5047 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/wal
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/data
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/logs
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/data/info.pb
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:53:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.253:45765
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.253
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:53:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:53:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:53:52.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44969/jwks.json with env {}
02:53:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.233105 5128 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } has no permanent_uuid. Determining permanent_uuid...
02:53:52.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.262084 5128 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } has no permanent_uuid. Determining permanent_uuid...
02:53:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.264523 5077 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.237.253:45765: connect: Connection refused (error 111)
02:53:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.268296 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.253:45765: Network error: Client connection negotiation failed: client connection to 127.4.237.253:45765: connect: Connection refused (error 111)
02:53:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.317740 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } attempt: 1
02:53:52.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.322091 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.253:45765: Network error: Client connection negotiation failed: client connection to 127.4.237.253:45765: connect: Connection refused (error 111)
02:53:52.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.390461 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } attempt: 2
02:53:52.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.394085 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.253:45765: Network error: Client connection negotiation failed: client connection to 127.4.237.253:45765: connect: Connection refused (error 111)
02:53:52.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.485478 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } attempt: 3
02:53:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.489032 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.253:45765: Network error: Client connection negotiation failed: client connection to 127.4.237.253:45765: connect: Connection refused (error 111)
02:53:52.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.526681 5130 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:53:52.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.527341 5130 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:53:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.527992 5130 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:53:52.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.559020 5130 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:53:52.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.559365 5130 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:53:52.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.559571 5130 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:53:52.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.559748 5130 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:53:52.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.559921 5130 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:53:52.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.560112 5130 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44969/jwks.json
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.596666 5130 master_runner.cc:386] Master server non-default flags:
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/data
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/wal
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.253:45765
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44969/jwks.json
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/data/info.pb
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.253
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:53:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:53:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:53:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/logs
02:53:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:53:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:53:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:53:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:53:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:53:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:53:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:53:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:53:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:53:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.597963 5130 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:53:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.600265 5130 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:53:52.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.618789 5140 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:52.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.632417 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } attempt: 4
02:53:52.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.636693 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.253:45765: Network error: Client connection negotiation failed: client connection to 127.4.237.253:45765: connect: Connection refused (error 111)
02:53:52.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.936061 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } attempt: 5
02:53:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.939797 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.253:45765: Network error: Client connection negotiation failed: client connection to 127.4.237.253:45765: connect: Connection refused (error 111)
02:53:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.487176 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } attempt: 6
02:53:53.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:53.493750 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.253:45765: Network error: Client connection negotiation failed: client connection to 127.4.237.253:45765: connect: Connection refused (error 111)
02:53:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.619638 5139 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:52.618806 5142 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:52.621346 5130 server_base.cc:1034] running on GCE node
02:53:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.721058 5130 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:53:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.724341 5130 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:53:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.725744 5130 hybrid_clock.cc:648] HybridClock initialized: now 1790650433725712 us; error 49 us; skew 500 ppm
02:53:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.726699 5130 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:53:53.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.736536 5130 webserver.cc:460] Webserver started at http://127.4.237.253:44299/ using document root <none> and password file <none>
02:53:53.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.737474 5130 fs_manager.cc:359] Metadata directory not provided
02:53:53.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.737653 5130 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:53:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.738032 5130 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:53:53.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.742650 5130 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/data/instance:
02:53:53.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fe28a1c823e436ca7c7b306a785ee17"
02:53:53.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:53 on dist-test-slave-sksk"
02:53:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.743723 5130 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/wal/instance:
02:53:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fe28a1c823e436ca7c7b306a785ee17"
02:53:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:53 on dist-test-slave-sksk"
02:53:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.750473 5130 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
02:53:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.755635 5151 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:53:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.756630 5130 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
02:53:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.756898 5130 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/wal
02:53:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fe28a1c823e436ca7c7b306a785ee17"
02:53:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:53 on dist-test-slave-sksk"
02:53:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.757234 5130 fs_report.cc:383] FS layout report
02:53:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:53:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/wal
02:53:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/wal
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/data/data
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:53:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:53:53.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.797550 5130 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:53:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.799012 5130 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:53:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.799449 5130 kserver.cc:163] Server-wide thread pool size limit: 3276
02:53:53.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.867424 5130 rpc_server.cc:307] RPC server started. Bound to: 127.4.237.253:45765
02:53:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.867528 5202 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.237.253:45765 every 8 connection(s)
02:53:53.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.869968 5130 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/data/info.pb
02:53:53.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.874871 5203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:53:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.879735 5047 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5130
02:53:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.880175 5047 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-1/wal/instance
02:53:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.883095 5047 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/wal
02:53:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/data
02:53:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:53:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:53:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:53:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/logs
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/data/info.pb
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.252:32841
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.252
02:53:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:53:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:53:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:53:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44969/jwks.json with env {}
02:53:53.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.888386 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } has no permanent_uuid. Determining permanent_uuid...
02:53:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.910408 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } has no permanent_uuid. Determining permanent_uuid...
02:53:53.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.926518 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } has no permanent_uuid. Determining permanent_uuid...
02:53:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:53.928776 5155 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:53.933900 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:53.983316 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 1
02:53:53.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:53.986981 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.055394 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 2
02:53:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.058849 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.150228 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 3
02:53:54.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.153872 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.181313 5206 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:53:54.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.181958 5206 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:53:54.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.182588 5206 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:53:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.212603 5206 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:53:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.212942 5206 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:53:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.213254 5206 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:53:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.213579 5206 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:53:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.213886 5206 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:53:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.214089 5206 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44969/jwks.json
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.248615 5206 master_runner.cc:386] Master server non-default flags:
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/data
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/wal
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.252:32841
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44969/jwks.json
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/data/info.pb
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.252
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/logs
02:53:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:53:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.250123 5206 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:53:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.252240 5206 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:53:54.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.269850 5216 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.297227 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 4
02:53:54.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.300767 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.554198 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } attempt: 7
02:53:54.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.576831 5128 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } has no permanent_uuid. Determining permanent_uuid...
02:53:54.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.582476 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.600312 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 5
02:53:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.603931 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.640985 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 1
02:53:54.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.644573 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.726060 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 2
02:53:54.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.729884 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.815405 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 3
02:53:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.819233 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.959654 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 4
02:53:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.963049 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.151419 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 6
02:53:55.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:55.157728 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.246603 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 5
02:53:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:55.250310 5128 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.237.252:32841: Network error: Client connection negotiation failed: client connection to 127.4.237.252:32841: connect: Connection refused (error 111)
02:53:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.269930 5217 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:54.271486 5206 server_base.cc:1034] running on GCE node
02:53:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:54.271281 5219 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.374257 5206 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:53:55.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.376809 5206 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:53:55.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.378145 5206 hybrid_clock.cc:648] HybridClock initialized: now 1790650435378099 us; error 62 us; skew 500 ppm
02:53:55.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.378929 5206 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:53:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.385581 5206 webserver.cc:460] Webserver started at http://127.4.237.252:41119/ using document root <none> and password file <none>
02:53:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.386445 5206 fs_manager.cc:359] Metadata directory not provided
02:53:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.386626 5206 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:53:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.386996 5206 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:53:55.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.393772 5206 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/data/instance:
02:53:55.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ea794bb1cfe48b98d8c4a54045203a5"
02:53:55.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:55 on dist-test-slave-sksk"
02:53:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.395249 5206 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/wal/instance:
02:53:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ea794bb1cfe48b98d8c4a54045203a5"
02:53:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:55 on dist-test-slave-sksk"
02:53:55.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.404281 5206 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
02:53:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.411008 5231 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:53:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.412092 5206 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
02:53:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.412359 5206 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/wal
02:53:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ea794bb1cfe48b98d8c4a54045203a5"
02:53:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:55 on dist-test-slave-sksk"
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.412658 5206 fs_report.cc:383] FS layout report
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/wal
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/wal
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/data/data
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:53:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:53:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.451160 5206 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:53:55.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.452603 5206 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:53:55.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.452988 5206 kserver.cc:163] Server-wide thread pool size limit: 3276
02:53:55.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.519866 5206 rpc_server.cc:307] RPC server started. Bound to: 127.4.237.252:32841
02:53:55.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.519974 5282 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.237.252:32841 every 8 connection(s)
02:53:55.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.522509 5206 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/data/info.pb
02:53:55.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.527463 5283 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:53:55.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.530890 5047 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5206
02:53:55.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.531324 5047 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/master-2/wal/instance
02:53:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.543571 5283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } has no permanent_uuid. Determining permanent_uuid...
02:53:55.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.573071 5283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } has no permanent_uuid. Determining permanent_uuid...
02:53:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.583907 5283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } has no permanent_uuid. Determining permanent_uuid...
02:53:55.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.600425 5283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5: Bootstrap starting.
02:53:55.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.606555 5283 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5: Neither blocks nor log segments found. Creating new log.
02:53:55.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.608240 5283 log.cc:824] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5: Log is configured to *not* fsync() on all Append() calls
02:53:55.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.612739 5283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5: No bootstrap required, opened a new log
02:53:55.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.633409 5283 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.634035 5283 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:53:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.634305 5283 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0ea794bb1cfe48b98d8c4a54045203a5, State: Initialized, Role: FOLLOWER
02:53:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.635213 5283 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.637941 5292 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.638787 5292 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: This master's current role is: FOLLOWER
02:53:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.639245 5283 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: configured and running, proceeding with master startup.
02:53:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.648501 5292 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:53:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.649088 5292 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.652112 5292 leader_election.cc:288] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 499b4c32ce7b4b1aad7164bef4c8b7a7 (127.4.237.254:41617), 4fe28a1c823e436ca7c7b306a785ee17 (127.4.237.253:45765)
02:53:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:55.667428 5303 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:53:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:55.667773 5303 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:53:55.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.662230 5206 master_runner.cc:186] Error getting master registration for 127.4.237.254:41617: OK, instance_id { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" instance_seqno: 1790650432191995 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:53:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.689749 5206 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:53:55.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:55.707885 5237 tablet.cc:2369] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:53:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.802749 5128 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 6
02:53:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.828745 5128 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: Bootstrap starting.
02:53:55.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.836073 5128 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: Neither blocks nor log segments found. Creating new log.
02:53:55.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.838726 5128 log.cc:824] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: Log is configured to *not* fsync() on all Append() calls
02:53:55.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.843891 5128 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: No bootstrap required, opened a new log
02:53:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.875087 5128 raft_consensus.cc:348] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.876346 5128 raft_consensus.cc:374] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:53:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.877345 5128 raft_consensus.cc:729] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 499b4c32ce7b4b1aad7164bef4c8b7a7, State: Initialized, Role: FOLLOWER
02:53:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.878440 5128 consensus_queue.cc:260] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.887111 5307 sys_catalog.cc:455] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:55.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.891053 5307 sys_catalog.cc:458] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [sys.catalog]: This master's current role is: FOLLOWER
02:53:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.890899 5103 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" is_pre_election: true
02:53:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.891988 5103 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0ea794bb1cfe48b98d8c4a54045203a5 in term 0.
02:53:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.890688 5128 sys_catalog.cc:564] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [sys.catalog]: configured and running, proceeding with master startup.
02:53:55.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.893332 5234 leader_election.cc:302] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ea794bb1cfe48b98d8c4a54045203a5, 499b4c32ce7b4b1aad7164bef4c8b7a7; no voters:
02:53:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.894311 5292 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:53:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.894644 5292 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:53:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.894939 5292 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 0 FOLLOWER]: Advancing to term 1
02:53:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:55.900571 5082 tablet.cc:2369] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:53:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.901765 5292 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.904333 5103 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7"
02:53:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.904889 5103 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [term 0 FOLLOWER]: Advancing to term 1
02:53:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.905717 5292 leader_election.cc:288] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [CANDIDATE]: Term 1 election: Requested vote from peers 499b4c32ce7b4b1aad7164bef4c8b7a7 (127.4.237.254:41617), 4fe28a1c823e436ca7c7b306a785ee17 (127.4.237.253:45765)
02:53:55.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.911502 5103 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0ea794bb1cfe48b98d8c4a54045203a5 in term 1.
02:53:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.912837 5234 leader_election.cc:302] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ea794bb1cfe48b98d8c4a54045203a5, 499b4c32ce7b4b1aad7164bef4c8b7a7; no voters:
02:53:55.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.916042 5292 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 1 FOLLOWER]: Leader election won for term 1
02:53:55.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.926440 5292 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [term 1 LEADER]: Becoming Leader. State: Replica: 0ea794bb1cfe48b98d8c4a54045203a5, State: Running, Role: LEADER
02:53:55.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.927574 5292 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.953992 5309 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0ea794bb1cfe48b98d8c4a54045203a5. Latest consensus state: current_term: 1 leader_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.954870 5309 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: This master's current role is: LEADER
02:53:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.959774 5314 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:53:55.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:55.968680 5314 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:53:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:56.012112 5322 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:53:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.013087 5103 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [term 1 FOLLOWER]: Refusing update from remote peer 0ea794bb1cfe48b98d8c4a54045203a5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:53:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:56.013221 5322 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:53:56.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.015040 5292 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:53:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.003319 5060 master_runner.cc:186] Error getting master registration for 127.4.237.253:45765: OK, instance_id { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" instance_seqno: 1790650433848607 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:53:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.099277 5307 sys_catalog.cc:455] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0ea794bb1cfe48b98d8c4a54045203a5. Latest consensus state: current_term: 1 leader_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.102059 5307 sys_catalog.cc:458] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [sys.catalog]: This master's current role is: FOLLOWER
02:53:56.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.103765 5060 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:53:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.113823 5324 mvcc.cc:205] Tried to move back new op lower bound from 7334504185869672448 to 7334504185569714176. Current Snapshot: MvccSnapshot[applied={T|T < 7334504185869672448}]
02:53:56.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.121838 5309 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:56.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.122573 5309 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: This master's current role is: LEADER
02:53:56.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.128075 5307 sys_catalog.cc:455] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.128796 5307 sys_catalog.cc:458] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7 [sys.catalog]: This master's current role is: FOLLOWER
02:53:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.135713 5314 catalog_manager.cc:1347] Generated new cluster ID: b83c82c578a54125982a588ce344f8cd
02:53:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.136013 5314 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:53:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.175056 5314 catalog_manager.cc:1370] Generated new certificate authority record
02:53:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.182106 5314 catalog_manager.cc:1504] Loading token signing keys...
02:53:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.218164 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } attempt: 7
02:53:56.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.236125 5314 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5: Generated new TSK 0
02:53:56.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.239153 5314 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:53:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.276033 5203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17: Bootstrap starting.
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.276783 5047 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/wal
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/data
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/logs
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/data/info.pb
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.193:0
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.237.193
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.193
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:53:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.284401 5203 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17: Neither blocks nor log segments found. Creating new log.
02:53:56.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.297341 5203 log.cc:824] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17: Log is configured to *not* fsync() on all Append() calls
02:53:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.314666 5203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17: No bootstrap required, opened a new log
02:53:56.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.344491 5203 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:56.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.345711 5203 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:53:56.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.346208 5203 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4fe28a1c823e436ca7c7b306a785ee17, State: Initialized, Role: FOLLOWER
02:53:56.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.347422 5203 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } }
02:53:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.358230 5331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:56.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.363157 5331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [sys.catalog]: This master's current role is: FOLLOWER
02:53:56.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.362747 5203 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [sys.catalog]: configured and running, proceeding with master startup.
02:53:56.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.364215 5177 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4fe28a1c823e436ca7c7b306a785ee17"
02:53:56.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.362921 5178 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4fe28a1c823e436ca7c7b306a785ee17" is_pre_election: true
02:53:56.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.363750 5176 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [term 0 FOLLOWER]: Advancing to term 1
02:53:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.374279 5176 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [term 1 FOLLOWER]: Refusing update from remote peer 0ea794bb1cfe48b98d8c4a54045203a5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:53:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.376437 5292 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:53:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.454464 5331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0ea794bb1cfe48b98d8c4a54045203a5. Latest consensus state: current_term: 1 leader_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.455235 5331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [sys.catalog]: This master's current role is: FOLLOWER
02:53:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.456395 5342 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:56.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.459486 5342 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17 [sys.catalog]: This master's current role is: FOLLOWER
02:53:56.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.483511 5347 catalog_manager.cc:1259] Loaded cluster ID: b83c82c578a54125982a588ce344f8cd
02:53:56.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.483796 5347 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17: loading cluster ID for follower catalog manager: success
02:53:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.497726 5347 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17: acquiring CA information for follower catalog manager: success
02:53:56.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.515461 5347 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 4fe28a1c823e436ca7c7b306a785ee17: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:53:56.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.861444 5292 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "499b4c32ce7b4b1aad7164bef4c8b7a7" member_type: VOTER last_known_addr { host: "127.4.237.254" port: 41617 } } peers { permanent_uuid: "4fe28a1c823e436ca7c7b306a785ee17" member_type: VOTER last_known_addr { host: "127.4.237.253" port: 45765 } } peers { permanent_uuid: "0ea794bb1cfe48b98d8c4a54045203a5" member_type: VOTER last_known_addr { host: "127.4.237.252" port: 32841 } } }
02:53:56.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:56.862540 5292 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ea794bb1cfe48b98d8c4a54045203a5 [sys.catalog]: This master's current role is: LEADER
02:53:56.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:56.913087 5329 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:53:56.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:56.913730 5329 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:53:56.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:56.914487 5329 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:53:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:56.970296 5329 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:53:56.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:56.971706 5329 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.237.193
02:53:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:57.019088 5322 catalog_manager.cc:1259] Loaded cluster ID: b83c82c578a54125982a588ce344f8cd
02:53:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:57.019865 5322 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: loading cluster ID for follower catalog manager: success
02:53:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:57.027860 5322 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: acquiring CA information for follower catalog manager: success
02:53:57.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:57.048131 5322 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 499b4c32ce7b4b1aad7164bef4c8b7a7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:53:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:57.056037 5329 tablet_server_runner.cc:78] Tablet server non-default flags:
02:53:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:53:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/data
02:53:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/wal
02:53:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.193:0
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/data/info.pb
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.193
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.237.193
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/logs
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:53:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:53:57.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:53:57.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:53:57.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:57.060091 5329 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:53:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:57.062628 5329 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:53:57.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:57.109607 5358 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:57.121843 5357 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:57.122769 5329 server_base.cc:1034] running on GCE node
02:53:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:57.113206 5361 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.611567 5329 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:53:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.614894 5329 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:53:58.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.616400 5329 hybrid_clock.cc:648] HybridClock initialized: now 1790650438616470 us; error 202 us; skew 500 ppm
02:53:58.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.617537 5329 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:53:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.625574 5329 webserver.cc:460] Webserver started at http://127.4.237.193:37621/ using document root <none> and password file <none>
02:53:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.626798 5329 fs_manager.cc:359] Metadata directory not provided
02:53:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.627076 5329 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:53:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.627665 5329 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:53:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.634661 5329 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/data/instance:
02:53:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ace7f4af46d8413e89a5b722a8f79ed9"
02:53:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:58 on dist-test-slave-sksk"
02:53:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.636193 5329 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/wal/instance:
02:53:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ace7f4af46d8413e89a5b722a8f79ed9"
02:53:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:58 on dist-test-slave-sksk"
02:53:58.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.645877 5329 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
02:53:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.654021 5367 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:53:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.655339 5329 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:53:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.655732 5329 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/wal
02:53:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ace7f4af46d8413e89a5b722a8f79ed9"
02:53:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:53:58 on dist-test-slave-sksk"
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.656183 5329 fs_report.cc:383] FS layout report
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/wal
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/wal
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/data/data
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:53:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:53:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.717604 5329 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:53:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.719555 5329 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:53:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.720117 5329 kserver.cc:163] Server-wide thread pool size limit: 3276
02:53:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.723529 5329 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:53:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.729336 5329 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:53:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.729631 5329 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:53:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.729952 5329 ts_tablet_manager.cc:610] Registered 0 tablets
02:53:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.730171 5329 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:53:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.895702 5329 rpc_server.cc:307] RPC server started. Bound to: 127.4.237.193:35319
02:53:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.895812 5481 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.237.193:35319 every 8 connection(s)
02:53:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.898860 5329 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/data/info.pb
02:53:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.907083 5047 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5329
02:53:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.907613 5047 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-0/wal/instance
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.920239 5047 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/wal
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/data
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/logs
02:53:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/data/info.pb
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.194:0
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.237.194
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.194
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:53:58.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.965554 5483 heartbeater.cc:347] Connected to a master server at 127.4.237.253:45765
02:53:58.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.966045 5483 heartbeater.cc:464] Registering TS with master...
02:53:58.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.966729 5484 heartbeater.cc:347] Connected to a master server at 127.4.237.254:41617
02:53:58.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.967056 5484 heartbeater.cc:464] Registering TS with master...
02:53:58.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.967917 5483 heartbeater.cc:511] Master 127.4.237.253:45765 requested a full tablet report, sending...
02:53:58.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.968040 5484 heartbeater.cc:511] Master 127.4.237.254:41617 requested a full tablet report, sending...
02:53:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.971411 5168 ts_manager.cc:194] Registered new tserver with Master: ace7f4af46d8413e89a5b722a8f79ed9 (127.4.237.193:35319)
02:53:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.973100 5482 heartbeater.cc:347] Connected to a master server at 127.4.237.252:32841
02:53:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.973415 5482 heartbeater.cc:464] Registering TS with master...
02:53:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.973515 5093 ts_manager.cc:194] Registered new tserver with Master: ace7f4af46d8413e89a5b722a8f79ed9 (127.4.237.193:35319)
02:53:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.974197 5482 heartbeater.cc:511] Master 127.4.237.252:32841 requested a full tablet report, sending...
02:53:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.976804 5248 ts_manager.cc:194] Registered new tserver with Master: ace7f4af46d8413e89a5b722a8f79ed9 (127.4.237.193:35319)
02:53:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:58.979574 5248 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.237.193:47395
02:53:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:59.243810 5489 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:53:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:59.244279 5489 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:53:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:59.244719 5489 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:53:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:59.275287 5489 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:53:59.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:59.276072 5489 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.237.194
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:59.309911 5489 tablet_server_runner.cc:78] Tablet server non-default flags:
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/data
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/wal
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.194:0
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/data/info.pb
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.194
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:53:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.237.194
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/logs
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:53:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:53:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:59.311123 5489 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:53:59.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:59.313210 5489 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:53:59.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:59.334702 5498 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:53:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:59.983829 5482 heartbeater.cc:503] Master 127.4.237.252:32841 was elected leader, sending a full tablet report...
02:54:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:59.334847 5501 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:53:59.334862 5499 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:53:59.336225 5489 server_base.cc:1034] running on GCE node
02:54:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.424153 5489 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:54:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.426661 5489 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:54:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.428068 5489 hybrid_clock.cc:648] HybridClock initialized: now 1790650440428044 us; error 47 us; skew 500 ppm
02:54:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.428817 5489 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:54:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.435201 5489 webserver.cc:460] Webserver started at http://127.4.237.194:42567/ using document root <none> and password file <none>
02:54:00.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.436061 5489 fs_manager.cc:359] Metadata directory not provided
02:54:00.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.436244 5489 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:54:00.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.436626 5489 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:54:00.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.441318 5489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/data/instance:
02:54:00.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01e073e89f70456bb0b1ec2546db15d3"
02:54:00.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:00 on dist-test-slave-sksk"
02:54:00.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.442337 5489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/wal/instance:
02:54:00.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01e073e89f70456bb0b1ec2546db15d3"
02:54:00.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:00 on dist-test-slave-sksk"
02:54:00.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.449124 5489 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
02:54:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.454370 5508 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:54:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.455453 5489 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.455775 5489 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/wal
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01e073e89f70456bb0b1ec2546db15d3"
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:00 on dist-test-slave-sksk"
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.456104 5489 fs_report.cc:383] FS layout report
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/wal
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/wal
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/data/data
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:54:00.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:54:00.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.498489 5489 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:54:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.499933 5489 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:54:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.500325 5489 kserver.cc:163] Server-wide thread pool size limit: 3276
02:54:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.502815 5489 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:54:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.506772 5489 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:54:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.506968 5489 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:54:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.507191 5489 ts_tablet_manager.cc:610] Registered 0 tablets
02:54:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.507328 5489 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:54:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.640408 5489 rpc_server.cc:307] RPC server started. Bound to: 127.4.237.194:43183
02:54:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.640507 5623 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.237.194:43183 every 8 connection(s)
02:54:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.642772 5489 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/data/info.pb
02:54:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.647900 5047 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5489
02:54:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.648586 5047 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-1/wal/instance
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.671914 5047 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/wal
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/data
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/logs
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/data/info.pb
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.195:0
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.237.195
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.195
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:54:00.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.701412 5625 heartbeater.cc:347] Connected to a master server at 127.4.237.253:45765
02:54:00.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.701929 5625 heartbeater.cc:464] Registering TS with master...
02:54:00.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.698779 5624 heartbeater.cc:347] Connected to a master server at 127.4.237.252:32841
02:54:00.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.702529 5624 heartbeater.cc:464] Registering TS with master...
02:54:00.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.703223 5625 heartbeater.cc:511] Master 127.4.237.253:45765 requested a full tablet report, sending...
02:54:00.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.703500 5624 heartbeater.cc:511] Master 127.4.237.252:32841 requested a full tablet report, sending...
02:54:00.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.705914 5247 ts_manager.cc:194] Registered new tserver with Master: 01e073e89f70456bb0b1ec2546db15d3 (127.4.237.194:43183)
02:54:00.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.706110 5167 ts_manager.cc:194] Registered new tserver with Master: 01e073e89f70456bb0b1ec2546db15d3 (127.4.237.194:43183)
02:54:00.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.707880 5247 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.237.194:47321
02:54:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.709794 5626 heartbeater.cc:347] Connected to a master server at 127.4.237.254:41617
02:54:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.710122 5626 heartbeater.cc:464] Registering TS with master...
02:54:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.711261 5626 heartbeater.cc:511] Master 127.4.237.254:41617 requested a full tablet report, sending...
02:54:00.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:00.713634 5093 ts_manager.cc:194] Registered new tserver with Master: 01e073e89f70456bb0b1ec2546db15d3 (127.4.237.194:43183)
02:54:01.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:01.009055 5634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:54:01.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:01.009523 5634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:54:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:01.009954 5634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:54:01.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:01.041119 5634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:54:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:01.041925 5634 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.237.195
02:54:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:01.076479 5634 tablet_server_runner.cc:78] Tablet server non-default flags:
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.237.212:35903
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/data
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/wal
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.237.195:0
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/data/info.pb
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.237.195
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.237.254:41617,127.4.237.253:45765,127.4.237.252:32841
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.237.195
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/logs
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:54:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:54:01.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:54:01.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:01.077729 5634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:54:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:01.079619 5634 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:54:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:01.098773 5641 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:01.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:01.712090 5624 heartbeater.cc:503] Master 127.4.237.252:32841 was elected leader, sending a full tablet report...
02:54:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:01.098861 5642 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:01.099466 5644 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:01.101576 5634 server_base.cc:1034] running on GCE node
02:54:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.187081 5634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:54:02.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.189249 5634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:54:02.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.190586 5634 hybrid_clock.cc:648] HybridClock initialized: now 1790650442190562 us; error 48 us; skew 500 ppm
02:54:02.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.191392 5634 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:54:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.197690 5634 webserver.cc:460] Webserver started at http://127.4.237.195:42695/ using document root <none> and password file <none>
02:54:02.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.198550 5634 fs_manager.cc:359] Metadata directory not provided
02:54:02.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.198733 5634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:54:02.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.199162 5634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:54:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.203991 5634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/data/instance:
02:54:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b48594ba7e6403fb3e42f01e7bf8370"
02:54:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:02 on dist-test-slave-sksk"
02:54:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.205060 5634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/wal/instance:
02:54:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b48594ba7e6403fb3e42f01e7bf8370"
02:54:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:02 on dist-test-slave-sksk"
02:54:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.212297 5634 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
02:54:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.217854 5651 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:54:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.218859 5634 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
02:54:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.219198 5634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/wal
02:54:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b48594ba7e6403fb3e42f01e7bf8370"
02:54:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:02 on dist-test-slave-sksk"
02:54:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.219524 5634 fs_report.cc:383] FS layout report
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/wal
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/wal
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/data/data
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:54:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:54:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.255456 5634 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:54:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.256891 5634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:54:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.257310 5634 kserver.cc:163] Server-wide thread pool size limit: 3276
02:54:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.259827 5634 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:54:02.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.263806 5634 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:54:02.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.263989 5634 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:54:02.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.264191 5634 ts_tablet_manager.cc:610] Registered 0 tablets
02:54:02.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.264318 5634 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:54:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.391507 5634 rpc_server.cc:307] RPC server started. Bound to: 127.4.237.195:43233
02:54:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.391614 5764 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.237.195:43233 every 8 connection(s)
02:54:02.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.393893 5634 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/data/info.pb
02:54:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.401567 5047 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5634
02:54:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.402037 5047 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster7293636819087787197/ts-2/wal/instance
02:54:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.439724 5765 heartbeater.cc:347] Connected to a master server at 127.4.237.252:32841
02:54:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.440191 5765 heartbeater.cc:464] Registering TS with master...
02:54:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.441272 5765 heartbeater.cc:511] Master 127.4.237.252:32841 requested a full tablet report, sending...
02:54:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.441382 5766 heartbeater.cc:347] Connected to a master server at 127.4.237.253:45765
02:54:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.441722 5766 heartbeater.cc:464] Registering TS with master...
02:54:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.442482 5766 heartbeater.cc:511] Master 127.4.237.253:45765 requested a full tablet report, sending...
02:54:02.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.443708 5247 ts_manager.cc:194] Registered new tserver with Master: 9b48594ba7e6403fb3e42f01e7bf8370 (127.4.237.195:43233)
02:54:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.445045 5167 ts_manager.cc:194] Registered new tserver with Master: 9b48594ba7e6403fb3e42f01e7bf8370 (127.4.237.195:43233)
02:54:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.445109 5247 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.237.195:47049
02:54:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.446080 5767 heartbeater.cc:347] Connected to a master server at 127.4.237.254:41617
02:54:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.446373 5767 heartbeater.cc:464] Registering TS with master...
02:54:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.447134 5767 heartbeater.cc:511] Master 127.4.237.254:41617 requested a full tablet report, sending...
02:54:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.449404 5093 ts_manager.cc:194] Registered new tserver with Master: 9b48594ba7e6403fb3e42f01e7bf8370 (127.4.237.195:43233)
02:54:02.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:02.467132 5047 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
02:54:02.477 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:54:02.478 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:54:02.488 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.237.254"
port: 41617
}
bound_http_address {
host: "127.4.237.254"
port: 46647
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.237.253"
port: 45765
}
bound_http_address {
host: "127.4.237.253"
port: 44299
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.237.252"
port: 32841
}
bound_http_address {
host: "127.4.237.252"
port: 41119
}
}
}
02:54:02.493 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:54:02.496 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.237.193"
port: 35319
}
bound_http_address {
host: "127.4.237.193"
port: 37621
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.237.194"
port: 43183
}
bound_http_address {
host: "127.4.237.194"
port: 42567
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.237.195"
port: 43233
}
bound_http_address {
host: "127.4.237.195"
port: 42695
}
}
}
02:54:02.496 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:54:02.686 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:54:02.793 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskL3tT2Z/test-tmp
02:54:02.793 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:54:02.793 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskL3tT2Z/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:54:02.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:54:03.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:03.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:54:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:03.123915 5777 mini_oidc.cc:152] Starting JWKS server
02:54:03.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:03.127473 5777 webserver.cc:460] Webserver started at http://127.0.0.1:41057/ using document root <none> and password file <none>
02:54:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:03.127980 5777 mini_oidc.cc:189] Starting OIDC Discovery server
02:54:03.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:03.129715 5777 webserver.cc:460] Webserver started at http://127.0.0.1:46729/ using document root <none> and password file <none>
02:54:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:54:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/principal' for realm 'KRBTEST.COM',
02:54:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:54:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:03 dist-test-slave-sksk krb5kdc[5789](info): setting up network...
02:54:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:54:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:03 dist-test-slave-sksk krb5kdc[5789](info): set up 2 sockets
02:54:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:03 dist-test-slave-sksk krb5kdc[5789](info): commencing operation
02:54:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:54:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:03.449373 5765 heartbeater.cc:503] Master 127.4.237.252:32841 was elected leader, sending a full tablet report...
02:54:04.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.158108 5777 mini_kdc.cc:121] Time spent starting KDC: real 1.003s user 0.003s sys 0.016s
02:54:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:54:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:54:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:54:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:54:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:54:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:54:04.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:54:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:54:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:54:04.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:54:04.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/test-user.keytab.
02:54:04.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/test-user.keytab.
02:54:04.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:04 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650444, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:04.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:54:04.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:54:04Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:54:04.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:54:04Z Disabled control of system clock
02:54:04.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.164.126@KRBTEST.COM; defaulting to no policy
02:54:04.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:54:04.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.164.126@KRBTEST.COM" created.
02:54:04.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:54:04.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.164.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.126.keytab.
02:54:04.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.164.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.126.keytab.
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:04.345625 5777 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/logs
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/info.pb
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.126:38623
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.126
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:54:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.126.keytab
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.126
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:54:04.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.644029 5803 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:54:04.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.644567 5803 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:54:04.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.645148 5803 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:54:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.677156 5803 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:54:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.677384 5803 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:54:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.677639 5803 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:54:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.677940 5803 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:54:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.678248 5803 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:54:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.678534 5803 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:54:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.678694 5803 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41057/jwks.json
02:54:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:04.714931 5803 master_runner.cc:386] Master server non-default flags:
02:54:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.126.keytab
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.126
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.126:38623
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/info.pb
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.126
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/logs
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:54:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:54:04.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:54:04.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:04.716069 5803 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:54:04.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:04.716984 5803 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:54:04.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.735160 5809 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.735154 5812 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:04.737401 5803 server_base.cc:1034] running on GCE node
02:54:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:04.735141 5810 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:05.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.844587 5803 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:54:05.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.847433 5803 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:54:05.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.848845 5803 hybrid_clock.cc:648] HybridClock initialized: now 1790650445848813 us; error 64 us; skew 500 ppm
02:54:05.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.849627 5803 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:54:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:05 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650445, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.858708 5803 init.cc:377] Logged in from keytab as kudu/127.5.164.126@KRBTEST.COM (short username kudu)
02:54:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.863507 5803 webserver.cc:460] Webserver started at http://127.5.164.126:43725/ using document root <none> and password file <none>
02:54:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.864368 5803 fs_manager.cc:359] Metadata directory not provided
02:54:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.864573 5803 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:54:05.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.865026 5803 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:54:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.869776 5803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/instance:
02:54:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5a776cacc11457fa19cf7b69fddc936"
02:54:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:05 on dist-test-slave-sksk"
02:54:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.871028 5803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal/instance:
02:54:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5a776cacc11457fa19cf7b69fddc936"
02:54:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:05 on dist-test-slave-sksk"
02:54:05.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.878492 5803 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
02:54:05.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.884382 5820 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:54:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.885401 5803 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.885685 5803 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5a776cacc11457fa19cf7b69fddc936"
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:05 on dist-test-slave-sksk"
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.885988 5803 fs_report.cc:383] FS layout report
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/data
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:54:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:54:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.926275 5803 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:54:05.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.931447 5803 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:54:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:05.931872 5803 kserver.cc:163] Server-wide thread pool size limit: 3276
02:54:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.007273 5803 rpc_server.cc:307] RPC server started. Bound to: 127.5.164.126:38623
02:54:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.007349 5871 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.164.126:38623 every 8 connection(s)
02:54:06.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.009831 5803 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/info.pb
02:54:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.014868 5777 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5803
02:54:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.015187 5872 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:54:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.015426 5777 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal/instance
02:54:06.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.028702 5872 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } has no permanent_uuid. Determining permanent_uuid...
02:54:06.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.164.125@KRBTEST.COM; defaulting to no policy
02:54:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:54:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.164.125@KRBTEST.COM" created.
02:54:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:06 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650445, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.126@KRBTEST.COM for kudu/127.5.164.126@KRBTEST.COM
02:54:06.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:54:06.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.164.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.125.keytab.
02:54:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.164.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.125.keytab.
02:54:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.059520 5777 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/logs
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/info.pb
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.125:42289
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.125
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.125.keytab
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.125
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:54:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.066303 5872 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } has no permanent_uuid. Determining permanent_uuid...
02:54:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.068444 5823 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.072124 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.125:42289: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.121744 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } attempt: 1
02:54:06.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.125559 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.125:42289: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.194015 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } attempt: 2
02:54:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.197595 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.125:42289: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.289036 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } attempt: 3
02:54:06.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.292462 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.125:42289: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.368831 5878 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:54:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.369366 5878 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:54:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.369978 5878 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:54:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.400625 5878 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:54:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.400866 5878 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:54:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.401149 5878 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:54:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.401355 5878 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:54:06.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.401548 5878 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:54:06.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.401777 5878 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:54:06.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.401954 5878 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41057/jwks.json
02:54:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.435962 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } attempt: 4
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.438246 5878 master_runner.cc:386] Master server non-default flags:
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.125.keytab
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.125
02:54:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.125:42289
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/info.pb
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.125
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/logs
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:54:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:54:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:54:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:54:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:54:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.439617 5878 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:54:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.439716 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.125:42289: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.440683 5878 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:54:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.459060 5885 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:06.739161 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } attempt: 5
02:54:06.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.742918 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.125:42289: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.300293 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } attempt: 6
02:54:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:07.306619 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.125:42289: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.459743 5888 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:06.463943 5886 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:07.606030 5887 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:54:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.606107 5878 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:54:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.609876 5878 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:54:07.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.612555 5878 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:54:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.613950 5878 hybrid_clock.cc:648] HybridClock initialized: now 1790650447613891 us; error 47 us; skew 500 ppm
02:54:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.614792 5878 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:54:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:07 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650447, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:07.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.620743 5878 init.cc:377] Logged in from keytab as kudu/127.5.164.125@KRBTEST.COM (short username kudu)
02:54:07.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.626240 5878 webserver.cc:460] Webserver started at http://127.5.164.125:44907/ using document root <none> and password file <none>
02:54:07.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.627418 5878 fs_manager.cc:359] Metadata directory not provided
02:54:07.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.627673 5878 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:54:07.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.628162 5878 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:54:07.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.633224 5878 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/instance:
02:54:07.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c88bc9290660459b9083b6d679c840ef"
02:54:07.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:07 on dist-test-slave-sksk"
02:54:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.634442 5878 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal/instance:
02:54:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c88bc9290660459b9083b6d679c840ef"
02:54:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:07 on dist-test-slave-sksk"
02:54:07.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.642930 5878 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
02:54:07.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.648736 5901 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:54:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.649852 5878 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:54:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.650241 5878 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c88bc9290660459b9083b6d679c840ef"
02:54:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:07 on dist-test-slave-sksk"
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.650614 5878 fs_report.cc:383] FS layout report
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/data
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:54:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:54:07.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.685973 5878 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:54:07.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.692023 5878 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:54:07.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.692533 5878 kserver.cc:163] Server-wide thread pool size limit: 3276
02:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.770381 5878 rpc_server.cc:307] RPC server started. Bound to: 127.5.164.125:42289
02:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.770440 5952 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.164.125:42289 every 8 connection(s)
02:54:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.773844 5878 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/info.pb
02:54:07.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.776958 5777 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5878
02:54:07.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.777443 5777 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal/instance
02:54:07.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.780507 5953 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:54:07.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.791304 5953 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } has no permanent_uuid. Determining permanent_uuid...
02:54:07.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.164.124@KRBTEST.COM; defaulting to no policy
02:54:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:54:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.164.124@KRBTEST.COM" created.
02:54:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:07 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650447, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.125@KRBTEST.COM for kudu/127.5.164.126@KRBTEST.COM
02:54:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:54:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.164.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.124.keytab.
02:54:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.164.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.124.keytab.
02:54:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.832327 5777 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/logs
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/info.pb
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.124:34373
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.124
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.124.keytab
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.124
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:54:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.837499 5953 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } has no permanent_uuid. Determining permanent_uuid...
02:54:07.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:07 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650447, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.125@KRBTEST.COM for kudu/127.5.164.125@KRBTEST.COM
02:54:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:07.856897 5868 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 5804
02:54:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.866097 5953 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } has no permanent_uuid. Determining permanent_uuid...
02:54:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:07.868734 5905 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:07.873471 5953 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.922997 5953 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 1
02:54:07.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:07.927006 5953 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:07.995438 5953 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 2
02:54:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:07.999418 5953 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:08.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.090848 5953 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 3
02:54:08.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.094686 5953 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.164749 5960 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:54:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.165377 5960 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:54:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.166152 5960 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:54:08.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.197611 5960 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:54:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.197862 5960 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:54:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.198273 5960 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:54:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.198598 5960 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:54:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.198899 5960 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:54:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.199200 5960 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:54:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.199404 5960 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41057/jwks.json
02:54:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.238112 5953 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 4
02:54:08.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.243281 5953 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.252651 5960 master_runner.cc:386] Master server non-default flags:
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.124.keytab
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.124
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.124:34373
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:54:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/info.pb
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.124
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/logs
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:54:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:54:08.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:54:08.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:54:08.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:54:08.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:54:08.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:54:08.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.253849 5960 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:54:08.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.254760 5960 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:54:08.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.276147 5969 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.367154 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } attempt: 7
02:54:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:08 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650445, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.126@KRBTEST.COM for kudu/127.5.164.125@KRBTEST.COM
02:54:08.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.437660 5872 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } has no permanent_uuid. Determining permanent_uuid...
02:54:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.461200 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:08.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.521209 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 1
02:54:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.542662 5953 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 5
02:54:08.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.544754 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:08.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.547429 5953 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.626148 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 2
02:54:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.632378 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:08.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.717868 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 3
02:54:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.725162 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:08.866465 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 4
02:54:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.874594 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:09.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.094825 5953 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 6
02:54:09.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:09.121867 5953 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.158047 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 5
02:54:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:09.163496 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.283252 5968 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:08.276830 5971 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:09.699396 5970 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1421 milliseconds
02:54:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.699504 5960 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:54:09.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.701011 5960 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:54:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.703959 5960 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:54:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.705287 5960 hybrid_clock.cc:648] HybridClock initialized: now 1790650449705253 us; error 41 us; skew 500 ppm
02:54:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.706094 5960 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:54:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:09 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650449, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.712361 5960 init.cc:377] Logged in from keytab as kudu/127.5.164.124@KRBTEST.COM (short username kudu)
02:54:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.715939 5872 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } attempt: 6
02:54:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.718192 5960 webserver.cc:460] Webserver started at http://127.5.164.124:35625/ using document root <none> and password file <none>
02:54:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.719520 5960 fs_manager.cc:359] Metadata directory not provided
02:54:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.719955 5960 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:54:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.720602 5960 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:54:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:09.722052 5872 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.164.124:34373: Network error: Client connection negotiation failed: client connection to 127.5.164.124:34373: connect: Connection refused (error 111)
02:54:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.727666 5960 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/instance:
02:54:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "546c5ce2e1894564acb6898bccbb4f4b"
02:54:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:09 on dist-test-slave-sksk"
02:54:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.729080 5960 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal/instance:
02:54:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "546c5ce2e1894564acb6898bccbb4f4b"
02:54:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:09 on dist-test-slave-sksk"
02:54:09.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.738826 5960 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
02:54:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.745592 5988 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:54:09.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.746699 5960 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
02:54:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.747000 5960 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "546c5ce2e1894564acb6898bccbb4f4b"
02:54:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:09 on dist-test-slave-sksk"
02:54:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.747324 5960 fs_report.cc:383] FS layout report
02:54:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:54:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/data
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:54:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:54:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.795801 5960 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:54:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.801649 5960 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:54:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.802105 5960 kserver.cc:163] Server-wide thread pool size limit: 3276
02:54:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.877146 5960 rpc_server.cc:307] RPC server started. Bound to: 127.5.164.124:34373
02:54:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.877199 6039 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.164.124:34373 every 8 connection(s)
02:54:09.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.880018 5960 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/info.pb
02:54:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.881996 5777 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 5960
02:54:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.882442 5777 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal/instance
02:54:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.892982 6041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:54:09.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.908563 6041 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } has no permanent_uuid. Determining permanent_uuid...
02:54:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:09 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650449, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.124@KRBTEST.COM for kudu/127.5.164.126@KRBTEST.COM
02:54:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:09 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650444, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.164.124@KRBTEST.COM
02:54:09.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.944161 6041 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } has no permanent_uuid. Determining permanent_uuid...
02:54:09.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:09 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650449, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.124@KRBTEST.COM for kudu/127.5.164.125@KRBTEST.COM
02:54:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:09 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650444, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.164.125@KRBTEST.COM
02:54:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:09.974109 6041 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } has no permanent_uuid. Determining permanent_uuid...
02:54:09.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:09 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650449, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.124@KRBTEST.COM for kudu/127.5.164.124@KRBTEST.COM
02:54:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:09 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650444, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.164.126@KRBTEST.COM
02:54:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.004127 6041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Bootstrap starting.
02:54:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.010243 6041 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Neither blocks nor log segments found. Creating new log.
02:54:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.011248 5777 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
02:54:10.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.012571 6041 log.cc:824] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Log is configured to *not* fsync() on all Append() calls
02:54:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.018293 6041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: No bootstrap required, opened a new log
02:54:10.021 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:10.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:54:10.036 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:54:10.037 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:54:10.041 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.5.164.126"
port: 38623
}
bound_http_address {
host: "127.5.164.126"
port: 43725
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.5.164.125"
port: 42289
}
bound_http_address {
host: "127.5.164.125"
port: 44907
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.5.164.124"
port: 34373
}
bound_http_address {
host: "127.5.164.124"
port: 35625
}
}
}
02:54:10.041 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:54:10.041 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:10.042 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:54:10.043 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:54:10.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.044505 6041 raft_consensus.cc:348] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:10.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.045894 6041 raft_consensus.cc:374] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:54:10.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.046180 6041 raft_consensus.cc:729] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 546c5ce2e1894564acb6898bccbb4f4b, State: Initialized, Role: FOLLOWER
02:54:10.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.046999 6041 consensus_queue.cc:260] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.050302 6050 sys_catalog.cc:455] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } } }
02:54:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.051048 6050 sys_catalog.cc:458] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [sys.catalog]: This master's current role is: FOLLOWER
02:54:10.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.052359 6041 sys_catalog.cc:564] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [sys.catalog]: configured and running, proceeding with master startup.
02:54:10.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.057670 5994 tablet.cc:2369] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:54:10.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:10 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650450, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:10.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:54:10.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:10.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.083577 6063 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:10.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.084084 6063 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:10.085 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc
02:54:10.093 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:54:10.094 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc
02:54:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.091595 6050 raft_consensus.cc:482] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:54:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.092116 6050 raft_consensus.cc:504] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:10.096 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:54:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.099844 6050 leader_election.cc:288] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623), c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289)
02:54:10.101 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:54:10.104 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:54:10.104 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.078182 5960 master_runner.cc:186] Error getting master registration for 127.5.164.126:38623: OK, instance_id { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" instance_seqno: 1790650445987877 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.112357 5960 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:54:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:10 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650450, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:54:10.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:10.146 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc
02:54:10.148 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:54:10.148 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.164.124:34373
02:54:10.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:54:10.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.151616 5777 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5960
02:54:10.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:10.181 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.164.125:42289
02:54:10.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:54:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.181876 5777 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5878
02:54:10.202 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:10.203 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.164.126:38623
02:54:10.204 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:54:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.204650 5777 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5803
02:54:10.224 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:10.224 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.164.124:34373
02:54:10.226 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.227665 5777 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/logs
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/info.pb
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.124:34373
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.124
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35625
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.124.keytab
02:54:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.124
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:10.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:54:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.542027 6067 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:54:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.542801 6067 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:54:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.543918 6067 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:54:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.582245 6067 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:54:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.582477 6067 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:54:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.582741 6067 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:54:10.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.582935 6067 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:54:10.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.583169 6067 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:54:10.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.583381 6067 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:54:10.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.583518 6067 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41057/jwks.json
02:54:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.620784 6067 master_runner.cc:386] Master server non-default flags:
02:54:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data
02:54:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.124.keytab
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.124
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.124:34373
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/info.pb
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.124
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35625
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/logs
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:54:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.622026 6067 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:54:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:10.623027 6067 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:54:10.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.640337 6075 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.640332 6074 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:11.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:10.641867 6077 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:11.825724 6076 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1184 milliseconds
02:54:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.825876 6067 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:54:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.827055 6067 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:54:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.829906 6067 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:54:11.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.831301 6067 hybrid_clock.cc:648] HybridClock initialized: now 1790650451831260 us; error 54 us; skew 500 ppm
02:54:11.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.832139 6067 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:54:11.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:11 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650451, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:11.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.842509 6067 init.cc:377] Logged in from keytab as kudu/127.5.164.124@KRBTEST.COM (short username kudu)
02:54:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.847347 6067 webserver.cc:460] Webserver started at http://127.5.164.124:35625/ using document root <none> and password file <none>
02:54:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.848229 6067 fs_manager.cc:359] Metadata directory not provided
02:54:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.848435 6067 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:54:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.856251 6067 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
02:54:11.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.860448 6088 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:54:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.861465 6067 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
02:54:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.861791 6067 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "546c5ce2e1894564acb6898bccbb4f4b"
02:54:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:09 on dist-test-slave-sksk"
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.864545 6067 fs_report.cc:383] FS layout report
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/wal
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/data
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:54:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:54:11.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.903457 6067 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:54:11.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.909161 6067 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:54:11.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.909569 6067 kserver.cc:163] Server-wide thread pool size limit: 3276
02:54:11.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.976949 6067 rpc_server.cc:307] RPC server started. Bound to: 127.5.164.124:34373
02:54:11.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.977085 6139 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.164.124:34373 every 8 connection(s)
02:54:11.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.979784 6067 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-2/data/info.pb
02:54:11.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.988068 5777 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 6067
02:54:11.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:11.988 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.164.125:42289
02:54:11.989 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.989974 6140 sys_catalog.cc:263] Verifying existing consensus state
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.990121 5777 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/logs
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/info.pb
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:54:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.125:42289
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.125
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44907
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.125.keytab
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.125
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:54:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:54:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:11.996995 6140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Bootstrap starting.
02:54:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.028641 6140 log.cc:824] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Log is configured to *not* fsync() on all Append() calls
02:54:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.033187 6140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:54:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.033789 6140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Bootstrap complete.
02:54:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.057790 6140 raft_consensus.cc:348] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.060233 6140 raft_consensus.cc:374] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:54:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.060436 6140 raft_consensus.cc:729] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 546c5ce2e1894564acb6898bccbb4f4b, State: Initialized, Role: FOLLOWER
02:54:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.061115 6140 consensus_queue.cc:260] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:12.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.063721 6144 sys_catalog.cc:455] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } } }
02:54:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.064743 6144 sys_catalog.cc:458] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [sys.catalog]: This master's current role is: FOLLOWER
02:54:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.064846 6140 sys_catalog.cc:564] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [sys.catalog]: configured and running, proceeding with master startup.
02:54:12.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.092702 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:12.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.093483 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.091583 6067 master_runner.cc:186] Error getting master registration for 127.5.164.126:38623: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111),
02:54:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.120530 6067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:54:12.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.164356 6094 tablet.cc:2369] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:54:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.261188 6144 raft_consensus.cc:482] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:54:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.261746 6144 raft_consensus.cc:504] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:12.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.266433 6089 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111)
02:54:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.268567 6144 leader_election.cc:288] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623), c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289)
02:54:12.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.275735 6089 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623): Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111)
02:54:12.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.277354 6091 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289): Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.277952 6091 leader_election.cc:302] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 546c5ce2e1894564acb6898bccbb4f4b; no voters: c88bc9290660459b9083b6d679c840ef, e5a776cacc11457fa19cf7b69fddc936
02:54:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.279078 6144 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:54:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.313331 6142 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:54:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.313884 6142 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:54:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.314543 6142 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:54:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.347999 6142 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:54:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.348230 6142 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:54:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.348627 6142 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:54:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.348857 6142 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:54:12.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.349061 6142 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:54:12.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.349293 6142 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:54:12.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.349433 6142 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41057/jwks.json
02:54:12.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.386545 6142 master_runner.cc:386] Master server non-default flags:
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.125.keytab
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.125
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.125:42289
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/info.pb
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.125
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44907
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/logs
02:54:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.387923 6142 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:54:12.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:12.389125 6142 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:54:12.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.409633 6165 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:13.098415 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:13.099223 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.410805 6168 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:13.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:12.409883 6166 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:13.797284 6167 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1383 milliseconds
02:54:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.797437 6142 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:54:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.798995 6142 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:54:13.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.802207 6142 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:54:13.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.803807 6142 hybrid_clock.cc:648] HybridClock initialized: now 1790650453803772 us; error 37 us; skew 500 ppm
02:54:13.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.804872 6142 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:54:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:13 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650453, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.811786 6142 init.cc:377] Logged in from keytab as kudu/127.5.164.125@KRBTEST.COM (short username kudu)
02:54:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.816969 6142 webserver.cc:460] Webserver started at http://127.5.164.125:44907/ using document root <none> and password file <none>
02:54:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.817782 6142 fs_manager.cc:359] Metadata directory not provided
02:54:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.817967 6142 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:54:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.825886 6142 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
02:54:13.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.830119 6178 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:54:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.831351 6142 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:54:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.831619 6142 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c88bc9290660459b9083b6d679c840ef"
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:07 on dist-test-slave-sksk"
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.833878 6142 fs_report.cc:383] FS layout report
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/data
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:54:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:54:13.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:54:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.891983 6142 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:54:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.900038 6142 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:54:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.900622 6142 kserver.cc:163] Server-wide thread pool size limit: 3276
02:54:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.996412 6142 rpc_server.cc:307] RPC server started. Bound to: 127.5.164.125:42289
02:54:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:13.996495 6229 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.164.125:42289 every 8 connection(s)
02:54:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.000293 6142 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/data/info.pb
02:54:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.006573 5777 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 6142
02:54:14.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:14.008 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.164.126:38623
02:54:14.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:54:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.010340 5777 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu
02:54:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data
02:54:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:54:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/logs
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/info.pb
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.126:38623
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.126
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43725
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.126.keytab
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.126
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:54:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.012665 6230 sys_catalog.cc:263] Verifying existing consensus state
02:54:14.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 02:54:14.013444 6230 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:54:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.013856 6142 master.cc:537] Master@127.5.164.125:42289 shutting down...
02:54:14.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.070329 6142 master.cc:552] Master@127.5.164.125:42289 shutdown complete.
02:54:14.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:54:14.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.102569 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:14.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.102849 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.314847 6234 raft_consensus.cc:482] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:54:14.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.315367 6234 raft_consensus.cc:504] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:14.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.317325 6234 leader_election.cc:288] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623), c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289)
02:54:14.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.341614 6089 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623): Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111)
02:54:14.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.342465 6091 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289): Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:14.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.342900 6091 leader_election.cc:302] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 546c5ce2e1894564acb6898bccbb4f4b; no voters: c88bc9290660459b9083b6d679c840ef, e5a776cacc11457fa19cf7b69fddc936
02:54:14.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.343638 6234 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:54:14.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.409225 6232 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:54:14.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.409849 6232 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:54:14.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.410526 6232 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:54:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.442039 6232 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:54:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.442298 6232 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:54:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.442689 6232 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:54:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.442986 6232 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:54:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.443300 6232 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:54:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.443585 6232 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:54:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.443795 6232 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41057/jwks.json
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.482028 6232 master_runner.cc:386] Master server non-default flags:
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.164.84:39813
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.164.126:38623,127.5.164.125:42289,127.5.164.124:34373
02:54:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/krb5kdc/kudu_127.5.164.126.keytab
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.164.126
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.164.126:38623
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41057/jwks.json
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:54:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/info.pb
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.164.126
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43725
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/logs
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:54:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:54:14.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.483595 6232 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:54:14.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.484985 6232 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:54:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.506728 6244 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:15.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:15.105738 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:15.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:15.106007 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:15.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:14.507242 6232 server_base.cc:1034] running on GCE node
02:54:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.508028 6246 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:14.507436 6243 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:54:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.650128 6232 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:54:15.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.652575 6232 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:54:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.653923 6232 hybrid_clock.cc:648] HybridClock initialized: now 1790650455653879 us; error 53 us; skew 500 ppm
02:54:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.654738 6232 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:54:15.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:15 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650455, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:15.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.660307 6232 init.cc:377] Logged in from keytab as kudu/127.5.164.126@KRBTEST.COM (short username kudu)
02:54:15.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.665107 6232 webserver.cc:460] Webserver started at http://127.5.164.126:43725/ using document root <none> and password file <none>
02:54:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.665993 6232 fs_manager.cc:359] Metadata directory not provided
02:54:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.666190 6232 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:54:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.673775 6232 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
02:54:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.678310 6254 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:54:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.679459 6232 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.679786 6232 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data,/tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5a776cacc11457fa19cf7b69fddc936"
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:54:05 on dist-test-slave-sksk"
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.682163 6232 fs_report.cc:383] FS layout report
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/data
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:54:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:54:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:54:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:54:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:54:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:54:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:54:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:54:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:54:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.719197 6232 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:54:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.724855 6232 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:54:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.725273 6232 kserver.cc:163] Server-wide thread pool size limit: 3276
02:54:15.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.794386 6232 rpc_server.cc:307] RPC server started. Bound to: 127.5.164.126:38623
02:54:15.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.794546 6307 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.164.126:38623 every 8 connection(s)
02:54:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.797044 6232 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/data/info.pb
02:54:15.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.804062 5777 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu as pid 6232
02:54:15.804 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:54:15.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.809403 6308 sys_catalog.cc:263] Verifying existing consensus state
02:54:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 02:54:15.810335 6308 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:54:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.810751 6232 master.cc:537] Master@127.5.164.126:38623 shutting down...
02:54:15.860 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.164.124:34373(127.5.164.124:34373)] connecting to peer
02:54:15.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:15.879436 6232 master.cc:552] Master@127.5.164.126:38623 shutdown complete.
02:54:15.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskL3tT2Z/test-tmp/mini-kudu-cluster8451528587795248083/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:54:15.928 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:15.929 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:15.978 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.5.164.124:34373(127.5.164.124:34373)] Successfully connected to peer
02:54:15.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:15.981 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:15.982 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.037 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.037 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.055 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:54:16.065 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.5.164.124
02:54:16.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:16.109754 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:16.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:16.110008 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:16 dist-test-slave-sksk krb5kdc[5789](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650450, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.164.124@KRBTEST.COM
02:54:16.276 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xff3d889f, L:/127.0.0.1:54876 - R:/127.5.164.124:34373] using SASL/GSSAPI
02:54:16.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:16.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:16.315 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.315 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.315 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:16.316 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.319 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:16.330 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:16.332 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:16.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.335 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.335 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.336 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.336 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:16.350 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:16.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.353 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:16.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.355 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.356 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.356 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.357 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:16.370 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:16.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:16.374 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.374 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.375 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.375 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.376 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.376 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.376 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:16.410 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:16.412 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:16.414 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.414 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.414 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.416 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.416 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.417 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.417 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:16.470 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:16.471 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:16.474 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.474 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.475 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.476 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.477 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.477 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.477 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:16.590 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:16.592 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:16.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.595 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.595 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.595 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.596 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:16.850 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:16.851 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:16.853 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.854 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:16.855 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.855 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:16.855 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:17.090 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:17.091 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:17.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:17.093 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:17.095 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:17.095 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:17.095 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:17.095 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:17.096 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:17.112867 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:17.113126 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:17.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:17.122948 6067 master_runner.cc:186] Error getting master registration for 127.5.164.126:38623: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111),
02:54:17.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:17.123350 6067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:54:17.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:17.635139 6316 raft_consensus.cc:482] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:54:17.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:17.635543 6316 raft_consensus.cc:504] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:17.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:17.637195 6316 leader_election.cc:288] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623), c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289)
02:54:17.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:17.639947 6091 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111) [suppressed 3 similar messages]
02:54:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:17.643203 6091 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289): Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:17.643646 6089 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623): Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111)
02:54:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:17.643983 6089 leader_election.cc:302] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 546c5ce2e1894564acb6898bccbb4f4b; no voters: c88bc9290660459b9083b6d679c840ef, e5a776cacc11457fa19cf7b69fddc936
02:54:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:17.644557 6316 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:54:18.110 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:18.112 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:18.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:18.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:18.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:18.115 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:18.115 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:18.115 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:18.116 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:18.116676 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:18.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:18.116930 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:19.119853 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:19.120182 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:19.710 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:19.713 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:19.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:19.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:19.714 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:19.715 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:19.715 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:19.715 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:19.715 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:20.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:20.122980 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:20.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:20.123286 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:21.126678 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:21.127019 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:21.768198 6325 raft_consensus.cc:482] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:54:21.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:21.768640 6325 raft_consensus.cc:504] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:21.770154 6325 leader_election.cc:288] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623), c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289)
02:54:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:21.774978 6089 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623): Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111)
02:54:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:21.775552 6091 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289): Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:21.775836 6091 leader_election.cc:302] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 546c5ce2e1894564acb6898bccbb4f4b; no voters: c88bc9290660459b9083b6d679c840ef, e5a776cacc11457fa19cf7b69fddc936
02:54:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:21.776360 6325 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:54:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:21 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650461, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:21.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:21.846130 6084 init.cc:295] Successfully reacquired a new kerberos TGT
02:54:22.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:22.125533 6067 master_runner.cc:186] Error getting master registration for 127.5.164.126:38623: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111),
02:54:22.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:22.125914 6067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:54:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:22.129732 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:22.130045 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:23.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:23.132900 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:23.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:23.133251 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:23.490 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:23.491 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:23.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:23.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:23.493 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:23.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:23.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:23.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:23.494 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:24.136303 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:24.136555 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:25.139353 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:25.139597 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:25.610 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:25.611 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:25.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:25.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:25.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:25.613 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:25.614 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:25.614 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:25.614 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:26.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:26.051 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:26.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:26.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:26.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:26.054 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:26.054 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:26.054 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:26.055 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:26.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:26.142481 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:26.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:26.142753 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:26.285435 6136 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 6069
02:54:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:26.286082 6136 debug-util.cc:398] Leaking SignalData structure 0x7b08000b75a0 after lost signal to thread 6139
02:54:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:26.741251 6335 raft_consensus.cc:482] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:54:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:26.741638 6335 raft_consensus.cc:504] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:26.742961 6335 leader_election.cc:288] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623), c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289)
02:54:26.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:26.745012 6089 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111) [suppressed 3 similar messages]
02:54:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:26.747567 6089 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623): Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111)
02:54:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:26.748260 6091 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289): Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:26.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:26.748651 6091 leader_election.cc:302] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 546c5ce2e1894564acb6898bccbb4f4b; no voters: c88bc9290660459b9083b6d679c840ef, e5a776cacc11457fa19cf7b69fddc936
02:54:26.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:26.749325 6335 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:54:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:27.145618 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:27.145860 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:27.234084 6067 master_runner.cc:186] Error getting master registration for 127.5.164.126:38623: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111),
02:54:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:27.234539 6067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:54:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:28.148554 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:28.148816 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:28.430 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:28.431 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:28.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:28.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:28.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:28.434 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:28.434 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:28.435 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:28.435 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:29.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:29.151574 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:29.151845 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:29.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:29.671 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:29.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:29.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:29.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:29.674 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:29.674 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:29.675 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:29.675 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:30.154618 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:30.154901 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:30.990 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:30.991 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:30.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:30.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:30.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:30.995 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:30.995 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:30.995 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:30.995 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:31.158054 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:31.158385 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:31 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650471, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:31.848587 6084 init.cc:295] Successfully reacquired a new kerberos TGT
02:54:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:32.161093 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:32.161365 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:32.237804 6067 master_runner.cc:186] Error getting master registration for 127.5.164.126:38623: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111),
02:54:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:32.238193 6067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:54:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:33.164152 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:33.164409 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:33.770 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:33.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:33.773 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:33.774 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:33.774 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:33.775 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:33.775 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:33.775 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:33.776 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:34.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:34.167253 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:34.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:34.167507 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:34.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:34.261626 6351 raft_consensus.cc:482] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:54:34.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:34.262079 6351 raft_consensus.cc:504] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5a776cacc11457fa19cf7b69fddc936" member_type: VOTER last_known_addr { host: "127.5.164.126" port: 38623 } } peers { permanent_uuid: "c88bc9290660459b9083b6d679c840ef" member_type: VOTER last_known_addr { host: "127.5.164.125" port: 42289 } } peers { permanent_uuid: "546c5ce2e1894564acb6898bccbb4f4b" member_type: VOTER last_known_addr { host: "127.5.164.124" port: 34373 } }
02:54:34.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:34.263928 6351 leader_election.cc:288] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623), c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289)
02:54:34.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:34.265939 6089 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:54:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:34.268692 6091 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c88bc9290660459b9083b6d679c840ef (127.5.164.125:42289): Network error: Client connection negotiation failed: client connection to 127.5.164.125:42289: connect: Connection refused (error 111)
02:54:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:34.269127 6089 leader_election.cc:334] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e5a776cacc11457fa19cf7b69fddc936 (127.5.164.126:38623): Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111)
02:54:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:34.269487 6089 leader_election.cc:302] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 546c5ce2e1894564acb6898bccbb4f4b; no voters: c88bc9290660459b9083b6d679c840ef, e5a776cacc11457fa19cf7b69fddc936
02:54:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:34.270059 6351 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:54:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:35.170169 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:35.170429 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:36.173316 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:36.173574 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:36.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:36.271 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:36.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:36.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:36.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:36.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:36.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:36.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:36.274 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:36.675045 5279 debug-util.cc:398] Leaking SignalData structure 0x7b080007f3c0 after lost signal to thread 5211
02:54:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:36.675650 5279 debug-util.cc:398] Leaking SignalData structure 0x7b08000b98e0 after lost signal to thread 5282
02:54:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:37.176627 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:37.176898 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:37.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:37.240911 6067 master_runner.cc:186] Error getting master registration for 127.5.164.126:38623: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111),
02:54:37.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:37.241287 6067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:54:37.869 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:37.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:37.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:37.872 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:37.873 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:37.874 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:37.875 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:37.875 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:37.879 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:38.179819 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:38.180097 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:39.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:39.182801 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:39.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:39.183072 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:39.650 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:39.651 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:39.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:39.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:39.651 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:39.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:39.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:39.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:39.653 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:40.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:40.185813 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:40.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:40.186088 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:40.910 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:40.911 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:40.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:40.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:40.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:40.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:40.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:40.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:40.914 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:41.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:41.189553 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:41.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:41.189900 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:41.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:54:41 dist-test-slave-sksk krb5kdc[5789](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790650481, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.164.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:54:41.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:41.850911 6084 init.cc:295] Successfully reacquired a new kerberos TGT
02:54:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:42.192435 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:42.192684 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:42.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:42.244285 6067 master_runner.cc:186] Error getting master registration for 127.5.164.126:38623: Network error: Client connection negotiation failed: client connection to 127.5.164.126:38623: connect: Connection refused (error 111),
02:54:42.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:42.244690 6067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:54:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:43.195062 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:43.195333 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:44.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:44.197650 6155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 546c5ce2e1894564acb6898bccbb4f4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:54:44.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:54:44.197899 6155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:54:44.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.125:42289(127.5.164.125:42289)] connecting to peer
02:54:44.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.164.126:38623(127.5.164.126:38623)] connecting to peer
02:54:44.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:44.671 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.164.125:42289(127.5.164.125:42289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:44.671 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.125:42289: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289
02:54:44.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:44.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.126:38623(127.5.164.126:38623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:44.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.164.126:38623: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
02:54:44.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623; will retry
02:54:44.677 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28865), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [168ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [224ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [474ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [478ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [502ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [503ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [506ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [519ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [521ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [523ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [540ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [541ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [544ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [561ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [562ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [564ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [601ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [604ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [661ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [663ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [664ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [782ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [783ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1043ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1283ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2296ms] refreshing cache from master, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [2298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [2300ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [2301ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2303ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3896ms] refreshing cache from master, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [3902ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3902ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [7676ms] refreshing cache from master, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [7677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [7680ms] too many traces: truncated at 100 traces, deferred=Deferred@1887565461(state=PENDING, result=null, callback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981), errback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981)))
02:54:44.683 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.164.124:34373(127.5.164.124:34373)] disconnecting while in state READY
02:54:44.684 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.164.125:42289(127.5.164.125:42289)] disconnecting while in state TERMINATED
02:54:44.685 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.164.126:38623(127.5.164.126:38623)] disconnecting while in state TERMINATED
02:54:44.688 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:54:44.690 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.5.164.124:34373(127.5.164.124:34373)] handling channelInactive
02:54:44.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.164.124:34373(127.5.164.124:34373)] cleaning up while in state READY due to: connection closed
02:54:44.691 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:54:44.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:44.696435 5777 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 6067
02:54:45.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:54:45Z chronyd exiting
02:54:46.793 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:54:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:46.794844 5047 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5329
02:54:46.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:46.814710 5047 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5489
02:54:46.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:46.836701 5047 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5634
02:54:46.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:46.858908 5047 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5060
02:54:46.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:46.883517 5047 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5130
02:54:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:54:46.908134 5047 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskL3tT2Z/build/tsan/bin/kudu with pid 5206
02:54:47.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:54:47Z chronyd exiting
02:54:48.994 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28865), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [168ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [224ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [474ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [478ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [502ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [503ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [506ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [519ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [521ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [523ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [540ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [541ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [544ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [561ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [562ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [564ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [601ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [604ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [661ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [663ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [664ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [782ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [783ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1043ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1283ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2296ms] refreshing cache from master, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [2298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [2300ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [2301ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2303ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3896ms] refreshing cache from master, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [3902ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3902ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [7676ms] refreshing cache from master, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [7677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [7680ms] too many traces: truncated at 100 traces, deferred=Deferred@1887565461(state=PENDING, result=null, callback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981), errback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.164.126:38623
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28865), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [168ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [224ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [474ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [478ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [502ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [503ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [506ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [519ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [521ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [523ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [540ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [541ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [544ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [561ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [562ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [564ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [601ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [604ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [661ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [663ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [664ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [782ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [783ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1043ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1283ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2296ms] refreshing cache from master, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [2298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [2300ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [2301ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2303ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3896ms] refreshing cache from master, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [3902ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3902ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [7676ms] refreshing cache from master, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [7677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [7680ms] too many traces: truncated at 100 traces, deferred=Deferred@1887565461(state=PENDING, result=null, callback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981), errback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.164.126:38623
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
02:54:50.108 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
02:54:50.108 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28865), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [168ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [224ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [474ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [478ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [502ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [503ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [506ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [519ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [521ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [523ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [540ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [541ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [544ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [561ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [562ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [564ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [601ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [604ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [661ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [663ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [664ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [782ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [783ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1043ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1283ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2296ms] refreshing cache from master, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [2298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [2300ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [2301ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2303ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3896ms] refreshing cache from master, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [3902ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3902ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [7676ms] refreshing cache from master, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [7677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [7680ms] too many traces: truncated at 100 traces, deferred=Deferred@1887565461(state=PENDING, result=null, callback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981), errback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.164.126:38623
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28865), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [168ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [224ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [474ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [478ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [502ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [503ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [506ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [519ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [521ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [523ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [540ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [541ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [544ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [561ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [562ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [564ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [601ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [604ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [661ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [663ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [664ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [782ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [783ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1043ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [1283ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2296ms] refreshing cache from master, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [2297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [2298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [2300ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [2301ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [2303ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3896ms] refreshing cache from master, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.124:34373: OK, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.125:42289: Network error: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289, [3902ms] Sub RPC ConnectToMaster: received response from server master-127.5.164.126:38623: Network error: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [3902ms] delaying RPC due to: Service unavailable: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623, [7676ms] refreshing cache from master, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.124:34373, [7676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.125:42289, [7677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.164.126:38623, [7680ms] too many traces: truncated at 100 traces, deferred=Deferred@1887565461(state=PENDING, result=null, callback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981), errback=(continuation of Deferred@927434801 after retry RPC after error@659508606) -> (continuation of Deferred@1244785124 after retry RPC after error@905677673) -> (continuation of Deferred@1047960551 after retry RPC after error@1661928938) -> (continuation of Deferred@649379494 after retry RPC after error@1333643738) -> (continuation of Deferred@1163844209 after retry RPC after error@1583720293) -> (continuation of Deferred@1562862465 after retry RPC after error@2083865318) -> (continuation of Deferred@1326158403 after retry RPC after error@1009146440) -> (continuation of Deferred@1518017830 after retry RPC after error@210720509) -> (continuation of Deferred@528827557 after retry RPC after error@301971719) -> (continuation of Deferred@326042675 after retry RPC after error@1696694301) -> (continuation of Deferred@1345601648 after retry RPC after error@1022356720) -> (continuation of Deferred@1472870850 after retry RPC after error@1048738382) -> (continuation of Deferred@1288145454 after retry RPC after error@252471082) -> (continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.164.124:34373,127.5.164.125:42289,127.5.164.126:38623) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.125:42289(127.5.164.125:42289): Connection refused: /127.5.164.125:42289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.164.126:38623(127.5.164.126:38623): Connection refused: /127.5.164.126:38623
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.164.126:38623
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)