01:56:58.672 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:56:58.685 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3pBqiM/test-tmp
01:56:58.688 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:56:58.692 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3pBqiM/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:56:58.986 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:56:59.145 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:56:59.146 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:56:59.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.150363 25018 mini_oidc.cc:152] Starting JWKS server
01:56:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.154922 25018 webserver.cc:460] Webserver started at http://127.0.0.1:33639/ using document root <none> and password file <none>
01:56:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.155455 25018 mini_oidc.cc:189] Starting OIDC Discovery server
01:56:59.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.158097 25018 webserver.cc:460] Webserver started at http://127.0.0.1:45437/ using document root <none> and password file <none>
01:56:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:56:59Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:56:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:56:59Z Disabled control of system clock
01:56:59.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.250170 25018 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:56:59.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:56:59.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/wal
01:56:59.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/data
01:56:59.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:56:59.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:56:59.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:56:59.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:56:59.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:56:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:56:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/logs
01:56:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/data/info.pb
01:56:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:56:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:56:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:56:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:56:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.190:42017
01:56:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.190
01:56:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:56:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:56:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:56:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:56:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:56:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:56:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:56:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:56:59.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33639/jwks.json with env {}
01:56:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.616796 25030 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:56:59.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.617646 25030 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:56:59.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.618398 25030 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:56:59.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.653191 25030 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:56:59.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.653541 25030 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:56:59.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.653827 25030 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:56:59.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.654382 25030 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:56:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.654747 25030 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:56:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.655046 25030 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33639/jwks.json
01:56:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.696900 25030 master_runner.cc:386] Master server non-default flags:
01:56:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:56:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:56:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:56:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:56:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/data
01:56:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/wal
01:56:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:56:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:56:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:56:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:56:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:56:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.190:42017
01:56:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:56:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:56:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:56:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33639/jwks.json
01:56:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:56:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:56:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/data/info.pb
01:56:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.190
01:56:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/logs
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:56:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:56:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:56:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:56:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:56:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:56:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:56:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:56:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:56:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.698801 25030 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:56:59.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.701524 25030 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:56:59.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.730320 25036 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.731565 25039 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:56:59.730353 25037 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:56:59.733065 25030 server_base.cc:1034] running on GCE node
01:57:01.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.053550 25030 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.057446 25030 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:01.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.059006 25030 hybrid_clock.cc:648] HybridClock initialized: now 1782007021058939 us; error 99 us; skew 500 ppm
01:57:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.059979 25030 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:01.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.068897 25030 webserver.cc:460] Webserver started at http://127.24.110.190:46877/ using document root <none> and password file <none>
01:57:01.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.070152 25030 fs_manager.cc:359] Metadata directory not provided
01:57:01.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.070377 25030 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:01.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.070854 25030 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.076834 25030 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/data/instance:
01:57:01.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9bd3197bfd249ebab707513dda094e7"
01:57:01.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:01 on dist-test-slave-jjq6"
01:57:01.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.078325 25030 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/wal/instance:
01:57:01.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9bd3197bfd249ebab707513dda094e7"
01:57:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:01 on dist-test-slave-jjq6"
01:57:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.087504 25030 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
01:57:01.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.095867 25046 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:01.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.098281 25030 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.006s sys 0.000s
01:57:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.099319 25030 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/wal
01:57:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9bd3197bfd249ebab707513dda094e7"
01:57:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:01 on dist-test-slave-jjq6"
01:57:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.099931 25030 fs_report.cc:383] FS layout report
01:57:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/wal
01:57:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/wal
01:57:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/data/data
01:57:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:01.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.142294 25030 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.144145 25030 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.144757 25030 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:01.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.235571 25030 rpc_server.cc:307] RPC server started. Bound to: 127.24.110.190:42017
01:57:01.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.235745 25097 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.110.190:42017 every 8 connection(s)
01:57:01.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.238656 25030 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/data/info.pb
01:57:01.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.245040 25098 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:57:01.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.247890 25018 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25030
01:57:01.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.248517 25018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-0/wal/instance
01:57:01.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.254673 25018 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:01.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/wal
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/data
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/logs
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/data/info.pb
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:01.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:01.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.189:43559
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.189
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:01.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33639/jwks.json with env {}
01:57:01.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.265470 25098 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } has no permanent_uuid. Determining permanent_uuid...
01:57:01.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.303012 25098 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } has no permanent_uuid. Determining permanent_uuid...
01:57:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.306185 25048 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.110.189:43559: connect: Connection refused (error 111)
01:57:01.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.311383 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.189:43559: Network error: Client connection negotiation failed: client connection to 127.24.110.189:43559: connect: Connection refused (error 111)
01:57:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.361195 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } attempt: 1
01:57:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.366267 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.189:43559: Network error: Client connection negotiation failed: client connection to 127.24.110.189:43559: connect: Connection refused (error 111)
01:57:01.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.434818 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } attempt: 2
01:57:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.439481 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.189:43559: Network error: Client connection negotiation failed: client connection to 127.24.110.189:43559: connect: Connection refused (error 111)
01:57:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.531038 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } attempt: 3
01:57:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.536738 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.189:43559: Network error: Client connection negotiation failed: client connection to 127.24.110.189:43559: connect: Connection refused (error 111)
01:57:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.601774 25100 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:01.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.602357 25100 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:01.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.602825 25100 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:01.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.637444 25100 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:01.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.637748 25100 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:01.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.637938 25100 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:01.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.638116 25100 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:01.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.638327 25100 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:01.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.638482 25100 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33639/jwks.json
01:57:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.677855 25100 master_runner.cc:386] Master server non-default flags:
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/data
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/wal
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.189:43559
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33639/jwks.json
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/data/info.pb
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.189
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/logs
01:57:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:01.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:01.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:01.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:01.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:01.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:01.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:01.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:01.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.679214 25100 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:01.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.680306 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } attempt: 4
01:57:01.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.681634 25100 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.685750 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.189:43559: Network error: Client connection negotiation failed: client connection to 127.24.110.189:43559: connect: Connection refused (error 111)
01:57:01.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.715778 25112 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:01.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.985267 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } attempt: 5
01:57:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.991489 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.189:43559: Network error: Client connection negotiation failed: client connection to 127.24.110.189:43559: connect: Connection refused (error 111)
01:57:02.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:02.538980 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } attempt: 6
01:57:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:02.546921 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.189:43559: Network error: Client connection negotiation failed: client connection to 127.24.110.189:43559: connect: Connection refused (error 111)
01:57:03.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.715811 25110 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:01.719184 25100 server_base.cc:1034] running on GCE node
01:57:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:01.715826 25109 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.036864 25100 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.039780 25100 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:03.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.041249 25100 hybrid_clock.cc:648] HybridClock initialized: now 1782007023041162 us; error 110 us; skew 500 ppm
01:57:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.042567 25100 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.050715 25100 webserver.cc:460] Webserver started at http://127.24.110.189:45119/ using document root <none> and password file <none>
01:57:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.052412 25100 fs_manager.cc:359] Metadata directory not provided
01:57:03.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.052672 25100 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.053908 25100 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:03.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.060803 25100 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/data/instance:
01:57:03.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5fbf62a242ca4fffa7902a7ee0bddc71"
01:57:03.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:03 on dist-test-slave-jjq6"
01:57:03.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.062418 25100 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/wal/instance:
01:57:03.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5fbf62a242ca4fffa7902a7ee0bddc71"
01:57:03.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:03 on dist-test-slave-jjq6"
01:57:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.073952 25100 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.010s sys 0.001s
01:57:03.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.081559 25121 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:03.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.083412 25100 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.004s sys 0.000s
01:57:03.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.083912 25100 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/wal
01:57:03.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5fbf62a242ca4fffa7902a7ee0bddc71"
01:57:03.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:03 on dist-test-slave-jjq6"
01:57:03.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.084326 25100 fs_report.cc:383] FS layout report
01:57:03.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:03.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/wal
01:57:03.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/wal
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/data/data
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:03.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:03.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:03.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.151652 25100 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.153226 25100 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.153676 25100 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.238752 25100 rpc_server.cc:307] RPC server started. Bound to: 127.24.110.189:43559
01:57:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.238801 25172 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.110.189:43559 every 8 connection(s)
01:57:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.243389 25100 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/data/info.pb
01:57:03.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.250617 25173 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:57:03.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.250939 25018 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25100
01:57:03.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.251276 25018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-1/wal/instance
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.253748 25018 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/wal
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/data
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/logs
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/data/info.pb
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.188:45713
01:57:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.188
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33639/jwks.json with env {}
01:57:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.265606 25173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } has no permanent_uuid. Determining permanent_uuid...
01:57:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.299994 25173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } has no permanent_uuid. Determining permanent_uuid...
01:57:03.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.323621 25173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } has no permanent_uuid. Determining permanent_uuid...
01:57:03.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.327126 25123 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.336897 25173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.386807 25173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 1
01:57:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.391557 25173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.460186 25173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 2
01:57:03.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.465276 25173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.556814 25173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 3
01:57:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.561968 25173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.599845 25175 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.600581 25175 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.601378 25175 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.607560 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } attempt: 7
01:57:03.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.630282 25098 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } has no permanent_uuid. Determining permanent_uuid...
01:57:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.638339 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.643600 25175 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.643972 25175 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.644306 25175 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.644655 25175 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:03.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.645011 25175 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:03.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.645325 25175 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33639/jwks.json
01:57:03.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.685271 25175 master_runner.cc:386] Master server non-default flags:
01:57:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/data
01:57:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/wal
01:57:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.188:45713
01:57:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33639/jwks.json
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/data/info.pb
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.188
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/logs
01:57:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.687204 25175 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.689983 25175 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:03.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.697031 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 1
01:57:03.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.704893 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.705636 25173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 4
01:57:03.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.713265 25173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.721684 25190 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.786628 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 2
01:57:03.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.791780 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:03.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.877666 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 3
01:57:03.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.883662 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:04.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:04.012989 25173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 5
01:57:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:04.018704 25173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:04.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:04.024740 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 4
01:57:04.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:04.029762 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:04.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:04.313522 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 5
01:57:04.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:04.320542 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:04.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:04.566196 25173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 6
01:57:04.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:04.574245 25173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:04.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:04.873261 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 6
01:57:04.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:04.881551 25098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.110.188:45713: Network error: Client connection negotiation failed: client connection to 127.24.110.188:45713: connect: Connection refused (error 111)
01:57:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:03.726404 25175 server_base.cc:1034] running on GCE node
01:57:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.724438 25187 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:03.722987 25188 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.018847 25175 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.022318 25175 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.023813 25175 hybrid_clock.cc:648] HybridClock initialized: now 1782007025023756 us; error 96 us; skew 500 ppm
01:57:05.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.024993 25175 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:05.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.034054 25175 webserver.cc:460] Webserver started at http://127.24.110.188:34349/ using document root <none> and password file <none>
01:57:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.035179 25175 fs_manager.cc:359] Metadata directory not provided
01:57:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.035398 25175 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:05.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.035917 25175 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.041971 25175 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/data/instance:
01:57:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c993603677714d5b81bff87837c897fa"
01:57:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:05 on dist-test-slave-jjq6"
01:57:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.043349 25175 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/wal/instance:
01:57:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c993603677714d5b81bff87837c897fa"
01:57:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:05 on dist-test-slave-jjq6"
01:57:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.052258 25175 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
01:57:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.059294 25201 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.060810 25175 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
01:57:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.061390 25175 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/wal
01:57:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c993603677714d5b81bff87837c897fa"
01:57:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:05 on dist-test-slave-jjq6"
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.061807 25175 fs_report.cc:383] FS layout report
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/wal
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/wal
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/data/data
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.105888 25175 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.107687 25175 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.108278 25175 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:05.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.197713 25175 rpc_server.cc:307] RPC server started. Bound to: 127.24.110.188:45713
01:57:05.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.197849 25252 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.110.188:45713 every 8 connection(s)
01:57:05.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.202385 25175 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/data/info.pb
01:57:05.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.211210 25253 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:57:05.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.212271 25018 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25175
01:57:05.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.212806 25018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/master-2/wal/instance
01:57:05.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.232410 25253 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } has no permanent_uuid. Determining permanent_uuid...
01:57:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.281025 25253 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } has no permanent_uuid. Determining permanent_uuid...
01:57:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.298547 25253 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } has no permanent_uuid. Determining permanent_uuid...
01:57:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.324507 25253 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa: Bootstrap starting.
01:57:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.332152 25253 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa: Neither blocks nor log segments found. Creating new log.
01:57:05.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.334587 25253 log.cc:824] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa: Log is configured to *not* fsync() on all Append() calls
01:57:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.340898 25253 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa: No bootstrap required, opened a new log
01:57:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.366794 25253 raft_consensus.cc:348] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.367905 25253 raft_consensus.cc:374] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:57:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.368216 25253 raft_consensus.cc:729] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c993603677714d5b81bff87837c897fa, State: Initialized, Role: FOLLOWER
01:57:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.369210 25253 consensus_queue.cc:260] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.372084 25262 sys_catalog.cc:455] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.372797 25262 sys_catalog.cc:458] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: This master's current role is: FOLLOWER
01:57:05.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.373776 25253 sys_catalog.cc:564] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: configured and running, proceeding with master startup.
01:57:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.386811 25262 raft_consensus.cc:482] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:05.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.387414 25262 raft_consensus.cc:504] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.390765 25262 leader_election.cc:288] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9bd3197bfd249ebab707513dda094e7 (127.24.110.190:42017), 5fbf62a242ca4fffa7902a7ee0bddc71 (127.24.110.189:43559)
01:57:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:05.418646 25273 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:05.419000 25273 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:05.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.405733 25175 master_runner.cc:186] Error getting master registration for 127.24.110.190:42017: OK, instance_id { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" instance_seqno: 1782007021212985 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:57:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.443799 25175 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:05.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:05.616961 25207 tablet.cc:2369] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:57:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.634969 25173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 7
01:57:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.662252 25173 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71: Bootstrap starting.
01:57:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.672417 25173 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71: Neither blocks nor log segments found. Creating new log.
01:57:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.676230 25173 log.cc:824] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71: Log is configured to *not* fsync() on all Append() calls
01:57:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.684517 25173 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71: No bootstrap required, opened a new log
01:57:05.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.722150 25173 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:05.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.723176 25173 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:57:05.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.723515 25173 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5fbf62a242ca4fffa7902a7ee0bddc71, State: Initialized, Role: FOLLOWER
01:57:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.724581 25173 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.729774 25277 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:05.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.730808 25148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c993603677714d5b81bff87837c897fa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" is_pre_election: true
01:57:05.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.731392 25277 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [sys.catalog]: This master's current role is: FOLLOWER
01:57:05.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.731878 25148 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c993603677714d5b81bff87837c897fa in term 0.
01:57:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.730520 25173 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [sys.catalog]: configured and running, proceeding with master startup.
01:57:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.733824 25203 leader_election.cc:302] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5fbf62a242ca4fffa7902a7ee0bddc71, c993603677714d5b81bff87837c897fa; no voters:
01:57:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.734915 25262 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 0 FOLLOWER]: Leader pre-election won for term 1
01:57:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.735569 25262 raft_consensus.cc:482] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:57:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.735950 25262 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 0 FOLLOWER]: Advancing to term 1
01:57:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.751291 25262 raft_consensus.cc:504] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.763613 25262 leader_election.cc:288] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [CANDIDATE]: Term 1 election: Requested vote from peers a9bd3197bfd249ebab707513dda094e7 (127.24.110.190:42017), 5fbf62a242ca4fffa7902a7ee0bddc71 (127.24.110.189:43559)
01:57:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.764261 25148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c993603677714d5b81bff87837c897fa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71"
01:57:05.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.764886 25148 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [term 0 FOLLOWER]: Advancing to term 1
01:57:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.775624 25148 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c993603677714d5b81bff87837c897fa in term 1.
01:57:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.777309 25203 leader_election.cc:302] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5fbf62a242ca4fffa7902a7ee0bddc71, c993603677714d5b81bff87837c897fa; no voters:
01:57:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.778471 25262 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 1 FOLLOWER]: Leader election won for term 1
01:57:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.787318 25262 raft_consensus.cc:686] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [term 1 LEADER]: Becoming Leader. State: Replica: c993603677714d5b81bff87837c897fa, State: Running, Role: LEADER
01:57:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.790422 25262 consensus_queue.cc:237] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:05.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.808329 25284 sys_catalog.cc:455] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: SysCatalogTable state changed. Reason: New leader c993603677714d5b81bff87837c897fa. Latest consensus state: current_term: 1 leader_uuid: "c993603677714d5b81bff87837c897fa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:05.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.809240 25284 sys_catalog.cc:458] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: This master's current role is: LEADER
01:57:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.820457 25290 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:57:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.822665 25291 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:57:05.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.829332 25290 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:57:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.863417 25148 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [term 1 FOLLOWER]: Refusing update from remote peer c993603677714d5b81bff87837c897fa: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:57:05.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.865247 25284 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [LEADER]: Connected to new peer: Peer: permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:57:05.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.828758 25100 master_runner.cc:186] Error getting master registration for 127.24.110.190:42017: OK, instance_id { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" instance_seqno: 1782007021212985 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:57:05.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.867406 25100 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.894860 25277 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c993603677714d5b81bff87837c897fa. Latest consensus state: current_term: 1 leader_uuid: "c993603677714d5b81bff87837c897fa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.895715 25277 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [sys.catalog]: This master's current role is: FOLLOWER
01:57:05.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.903410 25292 mvcc.cc:205] Tried to move back new op lower bound from 7299100777916043264 to 7299100777649553408. Current Snapshot: MvccSnapshot[applied={T|T < 7299100777916043264}]
01:57:05.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.905474 25262 sys_catalog.cc:455] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c993603677714d5b81bff87837c897fa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:05.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.906342 25262 sys_catalog.cc:458] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: This master's current role is: LEADER
01:57:05.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.910111 25290 catalog_manager.cc:1347] Generated new cluster ID: 40a3561c99c64cf39bc381a6ad379d18
01:57:05.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.910522 25290 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:57:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.914983 25098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } attempt: 7
01:57:05.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.926862 25277 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c993603677714d5b81bff87837c897fa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:05.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.927670 25277 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71 [sys.catalog]: This master's current role is: FOLLOWER
01:57:05.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.950800 25098 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7: Bootstrap starting.
01:57:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.959779 25098 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7: Neither blocks nor log segments found. Creating new log.
01:57:05.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.962565 25098 log.cc:824] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7: Log is configured to *not* fsync() on all Append() calls
01:57:05.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.968278 25098 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7: No bootstrap required, opened a new log
01:57:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.989526 25290 catalog_manager.cc:1370] Generated new certificate authority record
01:57:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:05.992806 25290 catalog_manager.cc:1504] Loading token signing keys...
01:57:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.014031 25098 raft_consensus.cc:348] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.016084 25098 raft_consensus.cc:374] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:57:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.016633 25098 raft_consensus.cc:729] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a9bd3197bfd249ebab707513dda094e7, State: Initialized, Role: FOLLOWER
01:57:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.017923 25098 consensus_queue.cc:260] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } }
01:57:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.024508 25290 catalog_manager.cc:5868] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa: Generated new TSK 0
01:57:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.025997 25290 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:57:06.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.037335 25098 sys_catalog.cc:564] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [sys.catalog]: configured and running, proceeding with master startup.
01:57:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.037585 25073 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c993603677714d5b81bff87837c897fa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a9bd3197bfd249ebab707513dda094e7" is_pre_election: true
01:57:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.039127 25071 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [term 0 FOLLOWER]: Advancing to term 1
01:57:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.022961 25299 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.042214 25299 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [sys.catalog]: This master's current role is: FOLLOWER
01:57:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.061264 25018 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/wal
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/data
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/logs
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/data/info.pb
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.129:0
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.110.129
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.129
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:57:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.060983 25072 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c993603677714d5b81bff87837c897fa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a9bd3197bfd249ebab707513dda094e7"
01:57:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.071790 25071 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [term 1 FOLLOWER]: Refusing update from remote peer c993603677714d5b81bff87837c897fa: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:57:06.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.074669 25262 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [LEADER]: Connected to new peer: Peer: permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:57:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.143347 25299 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c993603677714d5b81bff87837c897fa. Latest consensus state: current_term: 1 leader_uuid: "c993603677714d5b81bff87837c897fa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.143939 25299 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [sys.catalog]: This master's current role is: FOLLOWER
01:57:06.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.145576 25311 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c993603677714d5b81bff87837c897fa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:06.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.146685 25311 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7 [sys.catalog]: This master's current role is: FOLLOWER
01:57:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.175668 25316 catalog_manager.cc:1259] Loaded cluster ID: 40a3561c99c64cf39bc381a6ad379d18
01:57:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.176940 25316 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7: loading cluster ID for follower catalog manager: success
01:57:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.187058 25316 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7: acquiring CA information for follower catalog manager: success
01:57:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.193821 25316 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a9bd3197bfd249ebab707513dda094e7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:57:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:06.606804 25304 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:06.607532 25304 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:06.608310 25304 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:06.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.645036 25262 sys_catalog.cc:455] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c993603677714d5b81bff87837c897fa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9bd3197bfd249ebab707513dda094e7" member_type: VOTER last_known_addr { host: "127.24.110.190" port: 42017 } } peers { permanent_uuid: "5fbf62a242ca4fffa7902a7ee0bddc71" member_type: VOTER last_known_addr { host: "127.24.110.189" port: 43559 } } peers { permanent_uuid: "c993603677714d5b81bff87837c897fa" member_type: VOTER last_known_addr { host: "127.24.110.188" port: 45713 } } }
01:57:06.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.646445 25262 sys_catalog.cc:458] T 00000000000000000000000000000000 P c993603677714d5b81bff87837c897fa [sys.catalog]: This master's current role is: LEADER
01:57:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:06.675240 25304 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:06.677242 25304 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.110.129
01:57:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.748879 25304 tablet_server_runner.cc:78] Tablet server non-default flags:
01:57:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/data
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/wal
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.129:0
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/data/info.pb
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.129
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:06.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:06.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:06.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:06.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.110.129
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/logs
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.751432 25304 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.755806 25304 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:06.792471 25325 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:06.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.839470 25291 catalog_manager.cc:1259] Loaded cluster ID: 40a3561c99c64cf39bc381a6ad379d18
01:57:06.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.840407 25291 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71: loading cluster ID for follower catalog manager: success
01:57:06.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.857409 25291 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71: acquiring CA information for follower catalog manager: success
01:57:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:06.875311 25291 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5fbf62a242ca4fffa7902a7ee0bddc71: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:57:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:08.178500 25324 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 25304
01:57:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:08.388437 25304 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.600s user 0.495s sys 0.925s
01:57:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:08.394804 25304 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.607s user 0.495s sys 0.925s
01:57:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:06.793376 25326 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.421646 25304 server_base.cc:1034] running on GCE node
01:57:08.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:08.427804 25333 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.429467 25304 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:08.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.437649 25304 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:08.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.439708 25304 hybrid_clock.cc:648] HybridClock initialized: now 1782007028439551 us; error 95 us; skew 500 ppm
01:57:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.440832 25304 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.473368 25304 webserver.cc:460] Webserver started at http://127.24.110.129:38635/ using document root <none> and password file <none>
01:57:08.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.474699 25304 fs_manager.cc:359] Metadata directory not provided
01:57:08.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.475006 25304 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:08.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.475566 25304 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.482846 25304 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/data/instance:
01:57:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13dea7072b064bfeb5937ccf8024de30"
01:57:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:08 on dist-test-slave-jjq6"
01:57:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.484404 25304 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/wal/instance:
01:57:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13dea7072b064bfeb5937ccf8024de30"
01:57:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:08 on dist-test-slave-jjq6"
01:57:08.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.503517 25304 fs_manager.cc:691] Time spent creating directory manager: real 0.018s user 0.004s sys 0.012s
01:57:08.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.522528 25338 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:08.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.524712 25304 fs_manager.cc:725] Time spent opening block manager: real 0.015s user 0.005s sys 0.007s
01:57:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.525413 25304 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/wal
01:57:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "13dea7072b064bfeb5937ccf8024de30"
01:57:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:08 on dist-test-slave-jjq6"
01:57:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.526013 25304 fs_report.cc:383] FS layout report
01:57:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/wal
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/wal
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/data/data
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:08.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.613755 25304 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:08.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.616331 25304 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:08.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.617057 25304 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.627206 25304 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:57:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.640460 25304 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:57:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.640822 25304 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:57:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.641158 25304 ts_tablet_manager.cc:610] Registered 0 tablets
01:57:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:08.641425 25304 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:57:09.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.620738 25452 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.110.129:44079 every 8 connection(s)
01:57:09.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.621382 25304 rpc_server.cc:307] RPC server started. Bound to: 127.24.110.129:44079
01:57:09.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.626021 25304 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/data/info.pb
01:57:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.634539 25018 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25304
01:57:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.635345 25018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-0/wal/instance
01:57:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.652136 25018 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/wal
01:57:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/data
01:57:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/logs
01:57:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/data/info.pb
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:57:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.130:0
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.110.130
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.130
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:57:09.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.736521 25456 heartbeater.cc:347] Connected to a master server at 127.24.110.189:43559
01:57:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.739637 25456 heartbeater.cc:464] Registering TS with master...
01:57:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.743177 25456 heartbeater.cc:511] Master 127.24.110.189:43559 requested a full tablet report, sending...
01:57:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.747705 25138 ts_manager.cc:194] Registered new tserver with Master: 13dea7072b064bfeb5937ccf8024de30 (127.24.110.129:44079)
01:57:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.755818 25453 heartbeater.cc:347] Connected to a master server at 127.24.110.188:45713
01:57:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.756560 25453 heartbeater.cc:464] Registering TS with master...
01:57:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.757599 25453 heartbeater.cc:511] Master 127.24.110.188:45713 requested a full tablet report, sending...
01:57:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.760630 25218 ts_manager.cc:194] Registered new tserver with Master: 13dea7072b064bfeb5937ccf8024de30 (127.24.110.129:44079)
01:57:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.763552 25218 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.110.129:47365
01:57:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.787508 25460 heartbeater.cc:347] Connected to a master server at 127.24.110.190:42017
01:57:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.787918 25460 heartbeater.cc:464] Registering TS with master...
01:57:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.788671 25460 heartbeater.cc:511] Master 127.24.110.190:42017 requested a full tablet report, sending...
01:57:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:09.792738 25063 ts_manager.cc:194] Registered new tserver with Master: 13dea7072b064bfeb5937ccf8024de30 (127.24.110.129:44079)
01:57:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:10.164343 25459 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:10.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:10.164844 25459 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:10.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:10.165547 25459 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:10.206617 25459 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:10.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:10.207966 25459 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.110.130
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:10.272621 25459 tablet_server_runner.cc:78] Tablet server non-default flags:
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/data
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/wal
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.130:0
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/data/info.pb
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.130
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.110.130
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/logs
01:57:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:10.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:10.274262 25459 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:10.276355 25459 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:10.305310 25472 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:10.769060 25453 heartbeater.cc:503] Master 127.24.110.188:45713 was elected leader, sending a full tablet report...
01:57:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:10.308732 25469 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:10.305517 25470 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:11.573833 25471 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:57:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.573980 25459 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:57:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.579288 25459 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.583962 25459 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.585630 25459 hybrid_clock.cc:648] HybridClock initialized: now 1782007031585545 us; error 104 us; skew 500 ppm
01:57:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.586935 25459 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.597245 25459 webserver.cc:460] Webserver started at http://127.24.110.130:37187/ using document root <none> and password file <none>
01:57:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.598583 25459 fs_manager.cc:359] Metadata directory not provided
01:57:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.598877 25459 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.599542 25459 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.605680 25459 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/data/instance:
01:57:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "caf42ce8924d4f64b6840c48b2f5c1a3"
01:57:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:11 on dist-test-slave-jjq6"
01:57:11.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.607244 25459 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/wal/instance:
01:57:11.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "caf42ce8924d4f64b6840c48b2f5c1a3"
01:57:11.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:11 on dist-test-slave-jjq6"
01:57:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.617651 25459 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.011s sys 0.000s
01:57:11.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.624891 25480 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.626398 25459 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.626726 25459 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/wal
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "caf42ce8924d4f64b6840c48b2f5c1a3"
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:11 on dist-test-slave-jjq6"
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.627065 25459 fs_report.cc:383] FS layout report
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/wal
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/wal
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/data/data
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:11.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.672940 25459 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:11.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.674852 25459 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:11.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.675462 25459 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:11.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.678932 25459 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:57:11.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.684125 25459 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:57:11.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.684429 25459 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:57:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.684831 25459 ts_tablet_manager.cc:610] Registered 0 tablets
01:57:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.685122 25459 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:57:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.852705 25459 rpc_server.cc:307] RPC server started. Bound to: 127.24.110.130:42109
01:57:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.853217 25592 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.110.130:42109 every 8 connection(s)
01:57:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.855875 25459 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/data/info.pb
01:57:11.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.864943 25018 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25459
01:57:11.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.865464 25018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-1/wal/instance
01:57:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.878175 25018 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/wal
01:57:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/data
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/logs
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/data/info.pb
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.131:0
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.110.131
01:57:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.131
01:57:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:57:11.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.901629 25593 heartbeater.cc:347] Connected to a master server at 127.24.110.188:45713
01:57:11.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.902288 25593 heartbeater.cc:464] Registering TS with master...
01:57:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.903694 25593 heartbeater.cc:511] Master 127.24.110.188:45713 requested a full tablet report, sending...
01:57:11.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.906781 25218 ts_manager.cc:194] Registered new tserver with Master: caf42ce8924d4f64b6840c48b2f5c1a3 (127.24.110.130:42109)
01:57:11.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.908975 25218 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.110.130:45475
01:57:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.914927 25594 heartbeater.cc:347] Connected to a master server at 127.24.110.189:43559
01:57:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.915352 25594 heartbeater.cc:464] Registering TS with master...
01:57:11.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.916352 25594 heartbeater.cc:511] Master 127.24.110.189:43559 requested a full tablet report, sending...
01:57:11.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.919261 25595 heartbeater.cc:347] Connected to a master server at 127.24.110.190:42017
01:57:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.919672 25595 heartbeater.cc:464] Registering TS with master...
01:57:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.919834 25138 ts_manager.cc:194] Registered new tserver with Master: caf42ce8924d4f64b6840c48b2f5c1a3 (127.24.110.130:42109)
01:57:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.920516 25595 heartbeater.cc:511] Master 127.24.110.190:42017 requested a full tablet report, sending...
01:57:11.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:11.923430 25063 ts_manager.cc:194] Registered new tserver with Master: caf42ce8924d4f64b6840c48b2f5c1a3 (127.24.110.130:42109)
01:57:12.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:12.248958 25603 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:12.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:12.249655 25603 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:12.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:12.250310 25603 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:12.286247 25603 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:12.287252 25603 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.110.131
01:57:12.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:12.327332 25603 tablet_server_runner.cc:78] Tablet server non-default flags:
01:57:12.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.110.148:46353
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/data
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/wal
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.110.131:0
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/data/info.pb
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.110.131
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.110.190:42017,127.24.110.189:43559,127.24.110.188:45713
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.110.131
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/logs
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:12.329074 25603 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:12.331220 25603 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:12.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:12.351686 25610 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:12.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:12.916216 25593 heartbeater.cc:503] Master 127.24.110.188:45713 was elected leader, sending a full tablet report...
01:57:13.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:12.352856 25612 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:12.355655 25603 server_base.cc:1034] running on GCE node
01:57:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:12.352962 25609 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.650293 25603 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:13.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.653225 25603 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.654726 25603 hybrid_clock.cc:648] HybridClock initialized: now 1782007033654676 us; error 47 us; skew 500 ppm
01:57:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.655794 25603 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:13.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.664131 25603 webserver.cc:460] Webserver started at http://127.24.110.131:40321/ using document root <none> and password file <none>
01:57:13.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.665661 25603 fs_manager.cc:359] Metadata directory not provided
01:57:13.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.665915 25603 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:13.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.666369 25603 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:13.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.672216 25603 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/data/instance:
01:57:13.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a38c11d0ef443269d29e8feba2aae09"
01:57:13.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:13 on dist-test-slave-jjq6"
01:57:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.673947 25603 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/wal/instance:
01:57:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a38c11d0ef443269d29e8feba2aae09"
01:57:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:13 on dist-test-slave-jjq6"
01:57:13.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.683312 25603 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.003s sys 0.004s
01:57:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.691226 25620 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.692849 25603 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
01:57:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.693312 25603 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/wal
01:57:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a38c11d0ef443269d29e8feba2aae09"
01:57:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:13 on dist-test-slave-jjq6"
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.693722 25603 fs_report.cc:383] FS layout report
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/wal
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/wal
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/data/data
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:13.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.742776 25603 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:13.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.744303 25603 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:13.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.744750 25603 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.747635 25603 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:57:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.752277 25603 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:57:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.752545 25603 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:57:13.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.752876 25603 ts_tablet_manager.cc:610] Registered 0 tablets
01:57:13.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.753098 25603 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:57:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.923308 25603 rpc_server.cc:307] RPC server started. Bound to: 127.24.110.131:33133
01:57:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.923539 25732 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.110.131:33133 every 8 connection(s)
01:57:13.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.928107 25603 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/data/info.pb
01:57:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.935190 25018 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25603
01:57:13.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.935752 25018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster1715228047024265764/ts-2/wal/instance
01:57:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.973165 25733 heartbeater.cc:347] Connected to a master server at 127.24.110.188:45713
01:57:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.973707 25733 heartbeater.cc:464] Registering TS with master...
01:57:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.974761 25734 heartbeater.cc:347] Connected to a master server at 127.24.110.189:43559
01:57:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.975143 25733 heartbeater.cc:511] Master 127.24.110.188:45713 requested a full tablet report, sending...
01:57:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.975160 25734 heartbeater.cc:464] Registering TS with master...
01:57:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.976872 25734 heartbeater.cc:511] Master 127.24.110.189:43559 requested a full tablet report, sending...
01:57:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.978674 25218 ts_manager.cc:194] Registered new tserver with Master: 6a38c11d0ef443269d29e8feba2aae09 (127.24.110.131:33133)
01:57:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.979816 25138 ts_manager.cc:194] Registered new tserver with Master: 6a38c11d0ef443269d29e8feba2aae09 (127.24.110.131:33133)
01:57:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.980794 25218 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.110.131:55693
01:57:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.985823 25735 heartbeater.cc:347] Connected to a master server at 127.24.110.190:42017
01:57:13.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.986768 25735 heartbeater.cc:464] Registering TS with master...
01:57:13.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.987550 25735 heartbeater.cc:511] Master 127.24.110.190:42017 requested a full tablet report, sending...
01:57:13.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:13.991099 25062 ts_manager.cc:194] Registered new tserver with Master: 6a38c11d0ef443269d29e8feba2aae09 (127.24.110.131:33133)
01:57:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:14.006738 25018 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:57:14.018 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:57:14.020 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:57:14.031 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.110.190"
port: 42017
}
bound_http_address {
host: "127.24.110.190"
port: 46877
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.110.189"
port: 43559
}
bound_http_address {
host: "127.24.110.189"
port: 45119
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.110.188"
port: 45713
}
bound_http_address {
host: "127.24.110.188"
port: 34349
}
}
}
01:57:14.037 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:57:14.040 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.24.110.129"
port: 44079
}
bound_http_address {
host: "127.24.110.129"
port: 38635
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.24.110.130"
port: 42109
}
bound_http_address {
host: "127.24.110.130"
port: 37187
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.24.110.131"
port: 33133
}
bound_http_address {
host: "127.24.110.131"
port: 40321
}
}
}
01:57:14.040 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:57:14.303 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:57:14.460 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3pBqiM/test-tmp
01:57:14.461 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:57:14.461 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3pBqiM/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:57:14.464 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:57:14.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:14.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:57:14.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:14.830497 25744 mini_oidc.cc:152] Starting JWKS server
01:57:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:14.834758 25744 webserver.cc:460] Webserver started at http://127.0.0.1:42369/ using document root <none> and password file <none>
01:57:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:14.835350 25744 mini_oidc.cc:189] Starting OIDC Discovery server
01:57:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:14.837409 25744 webserver.cc:460] Webserver started at http://127.0.0.1:46751/ using document root <none> and password file <none>
01:57:14.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:57:14.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/principal' for realm 'KRBTEST.COM',
01:57:14.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:57:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:14 dist-test-slave-jjq6 krb5kdc[25756](info): setting up network...
01:57:14.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:57:14.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:14 dist-test-slave-jjq6 krb5kdc[25756](info): set up 2 sockets
01:57:14.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:14 dist-test-slave-jjq6 krb5kdc[25756](info): commencing operation
01:57:14.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:57:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:14.987236 25733 heartbeater.cc:503] Master 127.24.110.188:45713 was elected leader, sending a full tablet report...
01:57:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:16.071521 25744 mini_kdc.cc:121] Time spent starting KDC: real 1.205s user 0.006s sys 0.017s
01:57:16.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:57:16.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:57:16.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:57:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:57:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:57:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:57:16.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:57:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:57:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:57:16.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:16.308332 25744 mini_kdc.cc:253] Time spent creating user principal joe-interloper: real 0.101s user 0.000s sys 0.006s
01:57:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:57:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/test-user.keytab.
01:57:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/test-user.keytab.
01:57:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:17.001646 25744 mini_kdc.cc:302] Time spent creating keytab for test-user: real 0.693s user 0.002s sys 0.005s
01:57:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:17 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007037, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:57:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:57:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:57:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:57:17Z Disabled control of system clock
01:57:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.36.62@KRBTEST.COM; defaulting to no policy
01:57:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:57:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.36.62@KRBTEST.COM" created.
01:57:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:57:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.36.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.62.keytab.
01:57:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.36.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.62.keytab.
01:57:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.526749 25744 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.25.36.62: real 1.459s user 0.002s sys 0.009s
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:18.529946 25744 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/logs
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/info.pb
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.62:41043
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.62
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.62.keytab
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.62
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:57:18.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.705665 25589 debug-util.cc:398] Leaking SignalData structure 0x7b0800044840 after lost signal to thread 25464
01:57:18.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.707202 25589 debug-util.cc:398] Leaking SignalData structure 0x7b08000bea40 after lost signal to thread 25592
01:57:18.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.919734 25771 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:18.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.920455 25771 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:18.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.921497 25771 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:18.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.968789 25771 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:57:18.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.969056 25771 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:18.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.969398 25771 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:18.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.969667 25771 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:18.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.969879 25771 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:18.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.970070 25771 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:18.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:18.970256 25771 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42369/jwks.json
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:19.020483 25771 master_runner.cc:386] Master server non-default flags:
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.62.keytab
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.62
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.62:41043
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/info.pb
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.62
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/logs
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:19.021997 25771 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:19.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:19.023425 25771 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:19.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:19.051308 25778 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:19.055994 25780 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:19.056097 25771 server_base.cc:1034] running on GCE node
01:57:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:19.053436 25777 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.486806 25771 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.490012 25771 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.491729 25771 hybrid_clock.cc:648] HybridClock initialized: now 1782007040491652 us; error 82 us; skew 500 ppm
01:57:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.492627 25771 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:20.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:20 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007040, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:20.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.498965 25771 init.cc:377] Logged in from keytab as kudu/127.25.36.62@KRBTEST.COM (short username kudu)
01:57:20.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.505453 25771 webserver.cc:460] Webserver started at http://127.25.36.62:37943/ using document root <none> and password file <none>
01:57:20.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.506472 25771 fs_manager.cc:359] Metadata directory not provided
01:57:20.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.506726 25771 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:20.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.507256 25771 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.513453 25771 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/instance:
01:57:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2694556e57640bba1b03b95ad25bfff"
01:57:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:20 on dist-test-slave-jjq6"
01:57:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.515451 25771 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal/instance:
01:57:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2694556e57640bba1b03b95ad25bfff"
01:57:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:20 on dist-test-slave-jjq6"
01:57:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.526250 25771 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.005s
01:57:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.536271 25791 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:20.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.538184 25771 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.005s sys 0.002s
01:57:20.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.538738 25771 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:20.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2694556e57640bba1b03b95ad25bfff"
01:57:20.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:20 on dist-test-slave-jjq6"
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.539353 25771 fs_report.cc:383] FS layout report
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/data
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:20.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:20.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.597436 25771 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.606731 25771 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.607316 25771 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:20.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.703629 25771 rpc_server.cc:307] RPC server started. Bound to: 127.25.36.62:41043
01:57:20.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.703832 25842 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.36.62:41043 every 8 connection(s)
01:57:20.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.706634 25771 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/info.pb
01:57:20.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.709224 25744 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25771
01:57:20.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.709810 25744 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal/instance
01:57:20.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.715615 25843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:57:20.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.36.61@KRBTEST.COM; defaulting to no policy
01:57:20.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:57:20.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.36.61@KRBTEST.COM" created.
01:57:20.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.737412 25843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } has no permanent_uuid. Determining permanent_uuid...
01:57:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:57:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.36.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.61.keytab.
01:57:20.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.36.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.61.keytab.
01:57:20.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:20 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007040, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.62@KRBTEST.COM for kudu/127.25.36.62@KRBTEST.COM
01:57:20.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.777518 25744 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/logs
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/info.pb
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.61:42857
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.61
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.61.keytab
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.61
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:57:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.796854 25843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } has no permanent_uuid. Determining permanent_uuid...
01:57:20.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:20.799752 25793 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:20.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:20.805809 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.61:42857: Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:20.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.855474 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } attempt: 1
01:57:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:20.860217 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.61:42857: Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:20.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:20.928728 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } attempt: 2
01:57:20.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:20.932756 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.61:42857: Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:21.024333 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } attempt: 3
01:57:21.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.028607 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.61:42857: Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:21.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.132668 25849 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:21.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.133405 25849 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.134181 25849 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.168061 25849 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:57:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.168470 25849 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.169155 25849 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.169533 25849 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.169884 25849 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.170207 25849 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.170434 25849 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42369/jwks.json
01:57:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:21.172158 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } attempt: 4
01:57:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.177177 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.61:42857: Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:21.212935 25849 master_runner.cc:386] Master server non-default flags:
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:57:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.61.keytab
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.61
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.61:42857
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/info.pb
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.61
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/logs
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:21.214380 25849 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:21.215549 25849 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.236235 25859 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:21.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:21.476649 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } attempt: 5
01:57:21.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.484195 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.61:42857: Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:22.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.032066 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } attempt: 6
01:57:22.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:22.042884 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.61:42857: Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:22.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.236249 25856 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:21.236589 25857 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:22.659278 25858 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1423 milliseconds
01:57:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.659464 25849 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:57:22.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.666054 25849 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.669301 25849 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:22.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.671118 25849 hybrid_clock.cc:648] HybridClock initialized: now 1782007042671083 us; error 102 us; skew 500 ppm
01:57:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.672318 25849 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:22 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007042, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.679293 25849 init.cc:377] Logged in from keytab as kudu/127.25.36.61@KRBTEST.COM (short username kudu)
01:57:22.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.685137 25849 webserver.cc:460] Webserver started at http://127.25.36.61:41105/ using document root <none> and password file <none>
01:57:22.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.686553 25849 fs_manager.cc:359] Metadata directory not provided
01:57:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.686826 25849 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.687228 25849 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.692219 25849 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/instance:
01:57:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60f6c218d7e646d5983e6cc16e33e2bb"
01:57:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:22 on dist-test-slave-jjq6"
01:57:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.693421 25849 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal/instance:
01:57:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60f6c218d7e646d5983e6cc16e33e2bb"
01:57:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:22 on dist-test-slave-jjq6"
01:57:22.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.703083 25849 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
01:57:22.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.709650 25871 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.710911 25849 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
01:57:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.711336 25849 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60f6c218d7e646d5983e6cc16e33e2bb"
01:57:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:22 on dist-test-slave-jjq6"
01:57:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.711777 25849 fs_report.cc:383] FS layout report
01:57:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/data
01:57:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.759961 25849 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.767714 25849 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.768322 25849 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.856523 25849 rpc_server.cc:307] RPC server started. Bound to: 127.25.36.61:42857
01:57:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.856593 25922 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.36.61:42857 every 8 connection(s)
01:57:22.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.860627 25849 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/info.pb
01:57:22.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.863212 25744 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25849
01:57:22.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.863752 25744 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal/instance
01:57:22.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.868842 25923 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:57:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.36.60@KRBTEST.COM; defaulting to no policy
01:57:22.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.882712 25923 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } has no permanent_uuid. Determining permanent_uuid...
01:57:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:57:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.36.60@KRBTEST.COM" created.
01:57:22.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:22 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007042, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.61@KRBTEST.COM for kudu/127.25.36.62@KRBTEST.COM
01:57:22.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:57:22.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.36.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.60.keytab.
01:57:22.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.36.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.60.keytab.
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.922763 25744 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/logs
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/info.pb
01:57:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.60:40959
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.60
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.60.keytab
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.60
01:57:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:57:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.932370 25923 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } has no permanent_uuid. Determining permanent_uuid...
01:57:22.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:22 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007042, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.61@KRBTEST.COM for kudu/127.25.36.61@KRBTEST.COM
01:57:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:22.970427 25923 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } has no permanent_uuid. Determining permanent_uuid...
01:57:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:22.974118 25874 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:22.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:22.979713 25923 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.029268 25923 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 1
01:57:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.034375 25923 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.103003 25923 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 2
01:57:23.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.103576 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } attempt: 7
01:57:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.109452 25923 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:23 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007040, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.62@KRBTEST.COM for kudu/127.25.36.61@KRBTEST.COM
01:57:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.137209 25843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } has no permanent_uuid. Determining permanent_uuid...
01:57:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.143708 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.200995 25923 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 3
01:57:23.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.202217 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 1
01:57:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.206264 25923 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.209188 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.288769 25929 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.289587 25929 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.290452 25929 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.290707 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 2
01:57:23.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.295568 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.327261 25929 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:57:23.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.327526 25929 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:23.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.327785 25929 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.327987 25929 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.328177 25929 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.328348 25929 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.328492 25929 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42369/jwks.json
01:57:23.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.349748 25923 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 4
01:57:23.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.354293 25923 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.373239 25929 master_runner.cc:386] Master server non-default flags:
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.60.keytab
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.60
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.60:40959
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/info.pb
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.60
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/logs
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.374924 25929 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:23.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.376189 25929 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.381018 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 3
01:57:23.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.387135 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.400432 25940 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.527611 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 4
01:57:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.531957 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.653738 25923 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 5
01:57:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.661366 25923 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.815640 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 5
01:57:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.820552 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:24.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.209023 25923 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 6
01:57:24.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:24.224740 25923 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.373051 25843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } attempt: 6
01:57:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:24.380848 25843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.36.60:40959: Network error: Client connection negotiation failed: client connection to 127.25.36.60:40959: connect: Connection refused (error 111)
01:57:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.400736 25939 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:23.400966 25942 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:23.402796 25929 server_base.cc:1034] running on GCE node
01:57:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.724265 25929 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.727924 25929 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.729362 25929 hybrid_clock.cc:648] HybridClock initialized: now 1782007044729298 us; error 87 us; skew 500 ppm
01:57:24.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.730382 25929 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:24 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007044, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.744812 25929 init.cc:377] Logged in from keytab as kudu/127.25.36.60@KRBTEST.COM (short username kudu)
01:57:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.752877 25929 webserver.cc:460] Webserver started at http://127.25.36.60:43043/ using document root <none> and password file <none>
01:57:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.755069 25929 fs_manager.cc:359] Metadata directory not provided
01:57:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.755357 25929 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:24.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.755848 25929 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.763641 25929 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/instance:
01:57:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0e886f7227744cc68534fb2cbd50721b"
01:57:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:24 on dist-test-slave-jjq6"
01:57:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.765846 25929 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal/instance:
01:57:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0e886f7227744cc68534fb2cbd50721b"
01:57:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:24 on dist-test-slave-jjq6"
01:57:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.777663 25929 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.011s sys 0.001s
01:57:24.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.785202 25956 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.787492 25929 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.004s sys 0.000s
01:57:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.787994 25929 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0e886f7227744cc68534fb2cbd50721b"
01:57:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:24 on dist-test-slave-jjq6"
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.788425 25929 fs_report.cc:383] FS layout report
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/data
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.839231 25929 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.847875 25929 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.848516 25929 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.942304 25929 rpc_server.cc:307] RPC server started. Bound to: 127.25.36.60:40959
01:57:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.942409 26007 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.36.60:40959 every 8 connection(s)
01:57:24.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.945533 25929 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/info.pb
01:57:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.951472 26008 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:57:24.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.956274 25744 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 25929
01:57:24.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.957015 25744 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal/instance
01:57:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:24.966204 26008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } has no permanent_uuid. Determining permanent_uuid...
01:57:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:24 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007044, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.60@KRBTEST.COM for kudu/127.25.36.62@KRBTEST.COM
01:57:25.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:24 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007037, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.36.60@KRBTEST.COM
01:57:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.016811 26008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } has no permanent_uuid. Determining permanent_uuid...
01:57:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:25 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007044, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.60@KRBTEST.COM for kudu/127.25.36.61@KRBTEST.COM
01:57:25.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:25 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007037, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.36.61@KRBTEST.COM
01:57:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.050681 26008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } has no permanent_uuid. Determining permanent_uuid...
01:57:25.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:25 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007044, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.60@KRBTEST.COM for kudu/127.25.36.60@KRBTEST.COM
01:57:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:25 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007037, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.36.62@KRBTEST.COM
01:57:25.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.086853 26008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: Bootstrap starting.
01:57:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.094409 26008 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: Neither blocks nor log segments found. Creating new log.
01:57:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.096872 26008 log.cc:824] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: Log is configured to *not* fsync() on all Append() calls
01:57:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.097476 25744 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:57:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.103955 26008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: No bootstrap required, opened a new log
01:57:25.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:25.114 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:57:25.127 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:57:25.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:57:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.131490 26008 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.132462 26008 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:57:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.132763 26008 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0e886f7227744cc68534fb2cbd50721b, State: Initialized, Role: FOLLOWER
01:57:25.134 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.36.62"
port: 41043
}
bound_http_address {
host: "127.25.36.62"
port: 37943
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.36.61"
port: 42857
}
bound_http_address {
host: "127.25.36.61"
port: 41105
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.36.60"
port: 40959
}
bound_http_address {
host: "127.25.36.60"
port: 43043
}
}
}
01:57:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.133854 26008 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:25.135 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:57:25.135 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:25.135 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:57:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.137133 26017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } } }
01:57:25.137 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:57:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.137894 26017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [sys.catalog]: This master's current role is: FOLLOWER
01:57:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.139475 26008 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [sys.catalog]: configured and running, proceeding with master startup.
01:57:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:25 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007045, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:57:25.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.177629 26030 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:25.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.177999 26030 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:25.184 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:25.187 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc
01:57:25.196 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:57:25.197 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc
01:57:25.199 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:57:25.203 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
01:57:25.205 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:57:25.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:57:25.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.174261 25929 master_runner.cc:186] Error getting master registration for 127.25.36.62:41043: OK, instance_id { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" instance_seqno: 1782007040678026 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:57:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.208863 25929 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:25.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:25 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007045, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:57:25.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:25.253 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc
01:57:25.255 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:57:25.255 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.36.61:42857
01:57:25.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:57:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.258450 25744 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25849
01:57:25.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.285640 26017 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:25.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.286336 26017 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:25.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:25.289 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.36.60:40959
01:57:25.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:57:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.290221 25744 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25929
01:57:25.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:25.321 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.36.62:41043
01:57:25.322 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:57:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.322319 25744 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25771
01:57:25.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:25.347 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.36.61:42857
01:57:25.349 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.351859 25744 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/logs
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/info.pb
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.61:42857
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.61
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41105
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.61.keytab
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.61
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:57:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.685943 26034 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.686708 26034 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.687368 26034 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.720232 26034 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:57:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.720480 26034 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.720752 26034 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.720939 26034 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.721169 26034 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.721354 26034 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.721498 26034 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42369/jwks.json
01:57:25.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.763247 26034 master_runner.cc:386] Master server non-default flags:
01:57:25.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.61.keytab
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.61
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.61:42857
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/info.pb
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.61
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41105
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/logs
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.764806 26034 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.766146 26034 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.788583 26040 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.789551 26043 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:25.791143 26034 server_base.cc:1034] running on GCE node
01:57:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:25.788630 26041 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.127521 26034 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.130777 26034 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:27.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.132367 26034 hybrid_clock.cc:648] HybridClock initialized: now 1782007047132317 us; error 51 us; skew 500 ppm
01:57:27.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.133769 26034 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:27 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007047, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.141590 26034 init.cc:377] Logged in from keytab as kudu/127.25.36.61@KRBTEST.COM (short username kudu)
01:57:27.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.148720 26034 webserver.cc:460] Webserver started at http://127.25.36.61:41105/ using document root <none> and password file <none>
01:57:27.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.149935 26034 fs_manager.cc:359] Metadata directory not provided
01:57:27.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.150192 26034 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:27.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.160544 26034 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.001s sys 0.004s
01:57:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.166229 26052 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.167745 26034 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.168197 26034 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60f6c218d7e646d5983e6cc16e33e2bb"
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:22 on dist-test-slave-jjq6"
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.171095 26034 fs_report.cc:383] FS layout report
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/data
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:27.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.222831 26034 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:27.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.232141 26034 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.232796 26034 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.327999 26034 rpc_server.cc:307] RPC server started. Bound to: 127.25.36.61:42857
01:57:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.328191 26103 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.36.61:42857 every 8 connection(s)
01:57:27.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.330963 26034 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/data/info.pb
01:57:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.336308 25744 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 26034
01:57:27.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:27.337 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.36.60:40959
01:57:27.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.339679 25744 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/logs
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/info.pb
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.60:40959
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.60
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43043
01:57:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.60.keytab
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.60
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:57:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.347033 26104 sys_catalog.cc:263] Verifying existing consensus state
01:57:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 01:57:27.348091 26104 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:57:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.348599 26034 master.cc:537] Master@127.25.36.61:42857 shutting down...
01:57:27.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.421963 26034 master.cc:552] Master@127.25.36.61:42857 shutdown complete.
01:57:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:57:27.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.692593 26106 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:27.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.693207 26106 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:27.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.694015 26106 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.727789 26106 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:57:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.728062 26106 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.728369 26106 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.728688 26106 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:27.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.729027 26106 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:27.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.729408 26106 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:27.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.729705 26106 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42369/jwks.json
01:57:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.778215 26106 master_runner.cc:386] Master server non-default flags:
01:57:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.60.keytab
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.60
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.60:40959
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/info.pb
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.60
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43043
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/logs
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.779441 26106 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.780435 26106 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:27.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.803020 26113 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.804250 26115 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:27.805120 26112 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:27.806196 26106 server_base.cc:1034] running on GCE node
01:57:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.160471 26106 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:29.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.164443 26106 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.165977 26106 hybrid_clock.cc:648] HybridClock initialized: now 1782007049165935 us; error 63 us; skew 500 ppm
01:57:29.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.166919 26106 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:29 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007049, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.174520 26106 init.cc:377] Logged in from keytab as kudu/127.25.36.60@KRBTEST.COM (short username kudu)
01:57:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.181247 26106 webserver.cc:460] Webserver started at http://127.25.36.60:43043/ using document root <none> and password file <none>
01:57:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.182487 26106 fs_manager.cc:359] Metadata directory not provided
01:57:29.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.182790 26106 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.193445 26106 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
01:57:29.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.199492 26124 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:29.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.201009 26106 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
01:57:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.201447 26106 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0e886f7227744cc68534fb2cbd50721b"
01:57:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:24 on dist-test-slave-jjq6"
01:57:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.204272 26106 fs_report.cc:383] FS layout report
01:57:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/wal
01:57:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/data
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:29.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.291249 26106 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:29.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.298051 26106 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:29.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.298527 26106 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:29.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.384574 26106 rpc_server.cc:307] RPC server started. Bound to: 127.25.36.60:40959
01:57:29.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.384802 26175 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.36.60:40959 every 8 connection(s)
01:57:29.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.387913 26106 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-2/data/info.pb
01:57:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.397230 25744 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 26106
01:57:29.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:29.399 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.36.62:41043
01:57:29.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:57:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.401229 25744 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu
01:57:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data
01:57:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/logs
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/info.pb
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.62:41043
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.62
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37943
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.62.keytab
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.62
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:57:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.401850 26176 sys_catalog.cc:263] Verifying existing consensus state
01:57:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.409303 26176 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: Bootstrap starting.
01:57:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.466933 26176 log.cc:824] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: Log is configured to *not* fsync() on all Append() calls
01:57:29.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.473968 26176 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:57:29.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.475095 26176 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: Bootstrap complete.
01:57:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.515861 26176 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.519635 26176 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:57:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.519979 26176 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0e886f7227744cc68534fb2cbd50721b, State: Initialized, Role: FOLLOWER
01:57:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.521028 26176 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.524457 26180 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } } }
01:57:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.525455 26180 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [sys.catalog]: This master's current role is: FOLLOWER
01:57:29.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.526459 26176 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [sys.catalog]: configured and running, proceeding with master startup.
01:57:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.556069 26130 tablet.cc:2369] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:57:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.560506 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:29.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.561174 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.563840 26106 master_runner.cc:186] Error getting master registration for 127.25.36.62:41043: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111),
01:57:29.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.595943 26106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:29.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.835654 26178 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.836879 26178 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:29.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.838354 26178 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:29.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.873435 26178 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:57:29.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.873731 26178 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.874131 26178 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.874455 26178 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.874853 26178 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.875170 26178 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.875437 26178 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42369/jwks.json
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.917663 26178 master_runner.cc:386] Master server non-default flags:
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.36.20:46139
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.36.62:41043,127.25.36.61:42857,127.25.36.60:40959
01:57:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/krb5kdc/kudu_127.25.36.62.keytab
01:57:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.36.62
01:57:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.36.62:41043
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42369/jwks.json
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/info.pb
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.36.62
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37943
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/logs
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.918951 26178 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.920665 26178 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:29.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.953042 26201 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:29.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.974357 26180 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.975176 26180 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:29.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.979730 26180 leader_election.cc:288] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043), 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857)
01:57:29.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.984623 26126 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111)
01:57:29.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.991915 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043): Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111)
01:57:29.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.992765 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857): Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.993348 26126 leader_election.cc:302] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0e886f7227744cc68534fb2cbd50721b; no voters: 60f6c218d7e646d5983e6cc16e33e2bb, b2694556e57640bba1b03b95ad25bfff
01:57:29.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:29.994606 26180 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:57:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:30.566274 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:30.567091 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.957790 26203 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:31.391078 26202 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:57:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.391187 26178 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:57:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:29.953904 26200 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:31.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.403627 26178 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.407876 26178 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.409772 26178 hybrid_clock.cc:648] HybridClock initialized: now 1782007051409633 us; error 142 us; skew 500 ppm
01:57:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.411264 26178 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:31 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007051, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.425947 26178 init.cc:377] Logged in from keytab as kudu/127.25.36.62@KRBTEST.COM (short username kudu)
01:57:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.433003 26178 webserver.cc:460] Webserver started at http://127.25.36.62:37943/ using document root <none> and password file <none>
01:57:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.434546 26178 fs_manager.cc:359] Metadata directory not provided
01:57:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.434849 26178 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.447104 26178 fs_manager.cc:709] Time spent opening directory manager: real 0.008s user 0.008s sys 0.000s
01:57:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.453836 26216 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:31.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.455919 26178 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
01:57:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.456400 26178 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data,/tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2694556e57640bba1b03b95ad25bfff"
01:57:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:57:20 on dist-test-slave-jjq6"
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.460310 26178 fs_report.cc:383] FS layout report
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/data
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.509225 26178 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.516265 26178 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.516708 26178 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:31.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:31.571743 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:31.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:31.572552 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.624140 26178 rpc_server.cc:307] RPC server started. Bound to: 127.25.36.62:41043
01:57:31.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.624328 26267 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.36.62:41043 every 8 connection(s)
01:57:31.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.629868 26178 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/data/info.pb
01:57:31.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.640352 25744 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu as pid 26178
01:57:31.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:57:31.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.647330 26268 sys_catalog.cc:263] Verifying existing consensus state
01:57:31.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 01:57:31.648373 26268 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:57:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.648903 26178 master.cc:537] Master@127.25.36.62:41043 shutting down...
01:57:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.674154 26272 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.675071 26272 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:31.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.686805 26272 leader_election.cc:288] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043), 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857)
01:57:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:31.700587 26273 negotiation.cc:337] Failed RPC negotiation. Trace:
01:57:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:57:31.687078 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.25.36.62:41043 (local address 127.0.0.1:56188)
01:57:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:57:31.694506 (+ 7428us) negotiation.cc:107] Waiting for socket to connect
01:57:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:57:31.694603 (+ 97us) client_negotiation.cc:174] Beginning negotiation
01:57:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:57:31.695470 (+ 867us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
01:57:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0621 01:57:31.698905 (+ 3435us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
01:57:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":6887,"thread_start_us":6506,"threads_started":1}
01:57:31.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:31.732215 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043): Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111)
01:57:31.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:31.736712 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857): Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:31.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.738101 26126 leader_election.cc:302] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0e886f7227744cc68534fb2cbd50721b; no voters: 60f6c218d7e646d5983e6cc16e33e2bb, b2694556e57640bba1b03b95ad25bfff
01:57:31.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.739686 26272 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:57:31.764 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:31.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:31.787854 26178 master.cc:552] Master@127.25.36.62:41043 shutdown complete.
01:57:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3pBqiM/test-tmp/mini-kudu-cluster4683318658757317241/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:57:31.858 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.36.60:40959(127.25.36.60:40959)] connecting to peer
01:57:31.861 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:31.935 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.25.36.60:40959(127.25.36.60:40959)] Successfully connected to peer
01:57:31.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:31.941 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:31.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:31.953 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:31.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:31.954 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.037 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
01:57:32.048 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.25.36.60
01:57:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:32 dist-test-slave-jjq6 krb5kdc[25756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007045, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.36.60@KRBTEST.COM
01:57:32.337 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xfc2c5dff, L:/127.0.0.1:49646 - R:/127.25.36.60:40959] using SASL/GSSAPI
01:57:32.358 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.370 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.379 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.382 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.382 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.383 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.386 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.410 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.413 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.415 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.415 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.415 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.415 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.416 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.416 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.418 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.430 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.432 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.434 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.435 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.436 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.437 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.437 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.471 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.475 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.475 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.475 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.476 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.476 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.476 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.509 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.512 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.516 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.516 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.516 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.517 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.520 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.523 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.550 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.552 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.554 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.554 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.555 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.561 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:32.576692 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:32.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:32.577047 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:32.610 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.613 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.616 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.616 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.617 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.617 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.621 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.671 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.677 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.681 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.682 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:32.890 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:32.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:32.894 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.894 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.894 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.896 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:32.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:32.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:33.029 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:33.031 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:33.034 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:33.034 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:33.035 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:33.037 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:33.037 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:33.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:33.038 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:33.329 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:33.332 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:33.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:33.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:33.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:33.340 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:33.340 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:33.340 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:33.341 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:33.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:33.580215 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:33.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:33.580554 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:34.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:34.583595 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:34.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:34.583909 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:34.599669 26106 master_runner.cc:186] Error getting master registration for 127.25.36.62:41043: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111),
01:57:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:34.600118 26106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:34.942466 26284 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:34.943429 26284 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:34.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:34.945305 26284 leader_election.cc:288] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043), 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857)
01:57:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:34.951329 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043): Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111)
01:57:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:34.951992 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857): Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:34.952376 26126 leader_election.cc:302] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0e886f7227744cc68534fb2cbd50721b; no voters: 60f6c218d7e646d5983e6cc16e33e2bb, b2694556e57640bba1b03b95ad25bfff
01:57:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:34.953243 26284 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:57:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:35.587121 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:35.587386 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:35.770 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:35.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:35.774 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:35.775 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:35.775 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:35.775 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:35.775 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:35.775 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:35.776 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:36.590667 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:36.590986 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:37.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:37.594162 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:37.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:37.594429 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:38.598855 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:38.599285 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:39.129 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:39.131 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:39.134 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.134 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.134 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.134 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.134 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.136 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:39 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007059, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:39.179472 26120 init.cc:295] Successfully reacquired a new kerberos TGT
01:57:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:39.520279 26291 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:39.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:39.520870 26291 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:39.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:39.523011 26291 leader_election.cc:288] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043), 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857)
01:57:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:39.526618 26126 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111) [suppressed 5 similar messages]
01:57:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:39.531114 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043): Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111)
01:57:39.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:39.531812 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857): Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:39.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:39.532120 26126 leader_election.cc:302] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0e886f7227744cc68534fb2cbd50721b; no voters: 60f6c218d7e646d5983e6cc16e33e2bb, b2694556e57640bba1b03b95ad25bfff
01:57:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:39.532828 26291 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:57:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:39.602730 26106 master_runner.cc:186] Error getting master registration for 127.25.36.62:41043: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111),
01:57:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:39.603382 26106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:39.603585 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:39.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:39.603819 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:39.770 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:39.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:39.773 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.774 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.775 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.776 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.777 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.778 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:39.810 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:39.812 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:39.814 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.814 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.815 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.815 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.815 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:39.815 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:39.818 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:40.607117 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:40.607434 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:41.611099 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:41.611424 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:42.614516 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:42.614913 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:43.618872 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:43.619212 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:43.890 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:43.891 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:43.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:43.895 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:43.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:43.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:43.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:43.896 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:43.897 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:44.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:44.408443 26301 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:44.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:44.408947 26301 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:44.410610 26301 leader_election.cc:288] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043), 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857)
01:57:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:44.416276 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043): Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111)
01:57:44.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:44.416806 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857): Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:44.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:44.417083 26126 leader_election.cc:302] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0e886f7227744cc68534fb2cbd50721b; no voters: 60f6c218d7e646d5983e6cc16e33e2bb, b2694556e57640bba1b03b95ad25bfff
01:57:44.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:44.417753 26301 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:57:44.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:44.606055 26106 master_runner.cc:186] Error getting master registration for 127.25.36.62:41043: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111),
01:57:44.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:44.606541 26106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:44.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:44.622388 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:44.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:44.622730 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:45.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:45.626139 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:45.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:45.626456 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:46.629817 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:46.630103 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:47.629 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:47.631 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:47.634115 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:47.634469 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:47.635 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:47.635 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:47.635 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:47.635 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:47.635 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:47.636 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:47.636 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:48.638394 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:48.638908 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:57:49 dist-test-slave-jjq6 krb5kdc[25756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782007069, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.36.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:57:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:49.182611 26120 init.cc:295] Successfully reacquired a new kerberos TGT
01:57:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:49.610868 26106 master_runner.cc:186] Error getting master registration for 127.25.36.62:41043: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111),
01:57:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:49.611546 26106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:49.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:49.642699 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:49.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:49.643078 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:50.310 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:50.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:50.314 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:50.315 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:50.316 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:50.316 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:50.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:50.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:50.319 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:50.325786 25094 debug-util.cc:398] Leaking SignalData structure 0x7b08000a0760 after lost signal to thread 25031
01:57:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:50.327004 25094 debug-util.cc:398] Leaking SignalData structure 0x7b080008efe0 after lost signal to thread 25097
01:57:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:50.646692 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:50.647047 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:51.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:51.650641 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:51.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:51.651135 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:52.569 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:52.571 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:52.573 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:52.573 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:52.573 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:52.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:52.574 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:52.574 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:52.576 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:52.654263 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:52.654578 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:53.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:53.337783 26314 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:53.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:53.338332 26314 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:53.340480 26314 leader_election.cc:288] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043), 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857)
01:57:53.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:53.343500 26126 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111) [suppressed 3 similar messages]
01:57:53.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:53.347066 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043): Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111)
01:57:53.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:53.347831 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857): Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:53.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:53.348177 26126 leader_election.cc:302] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0e886f7227744cc68534fb2cbd50721b; no voters: 60f6c218d7e646d5983e6cc16e33e2bb, b2694556e57640bba1b03b95ad25bfff
01:57:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:53.349157 26314 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:57:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:53.657968 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:53.658342 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:54.615072 26106 master_runner.cc:186] Error getting master registration for 127.25.36.62:41043: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111),
01:57:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:54.615679 26106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:57:54.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:54.661515 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:54.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:54.661816 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:55.329 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:55.331 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:55.333 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:55.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:55.334 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:55.334 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:55.334 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:55.334 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:55.337 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:55.370 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:55.371 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:55.373 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:55.373 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:55.373 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:55.374 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:55.374 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:55.374 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:55.377 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:55.666592 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:55.666952 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:56.666217 25169 debug-util.cc:398] Leaking SignalData structure 0x7b080009bfc0 after lost signal to thread 25104
01:57:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:56.667068 25169 debug-util.cc:398] Leaking SignalData structure 0x7b08000b95a0 after lost signal to thread 25172
01:57:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:56.672415 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:56.672915 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:57.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:57.678750 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:57.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:57.679181 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:58.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:58.682791 26191 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:57:58.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:58.683120 26191 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:57:58.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:58.724251 26323 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:57:58.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:58.724853 26323 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2694556e57640bba1b03b95ad25bfff" member_type: VOTER last_known_addr { host: "127.25.36.62" port: 41043 } } peers { permanent_uuid: "60f6c218d7e646d5983e6cc16e33e2bb" member_type: VOTER last_known_addr { host: "127.25.36.61" port: 42857 } } peers { permanent_uuid: "0e886f7227744cc68534fb2cbd50721b" member_type: VOTER last_known_addr { host: "127.25.36.60" port: 40959 } }
01:57:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:58.727063 26323 leader_election.cc:288] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043), 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857)
01:57:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:58.730942 26126 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:58.735471 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b2694556e57640bba1b03b95ad25bfff (127.25.36.62:41043): Network error: Client connection negotiation failed: client connection to 127.25.36.62:41043: connect: Connection refused (error 111)
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:57:58.736227 26126 leader_election.cc:334] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 60f6c218d7e646d5983e6cc16e33e2bb (127.25.36.61:42857): Network error: Client connection negotiation failed: client connection to 127.25.36.61:42857: connect: Connection refused (error 111)
01:57:58.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:58.736871 26126 leader_election.cc:302] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0e886f7227744cc68534fb2cbd50721b; no voters: 60f6c218d7e646d5983e6cc16e33e2bb, b2694556e57640bba1b03b95ad25bfff
01:57:58.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:58.737797 26323 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0e886f7227744cc68534fb2cbd50721b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:57:58.829 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:58.830 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:58.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:58.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:58.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:58.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:58.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:58.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:58.834 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:58.869 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.61:42857(127.25.36.61:42857)] connecting to peer
01:57:58.870 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.36.62:41043(127.25.36.62:41043)] connecting to peer
01:57:58.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:58.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.36.62:41043(127.25.36.62:41043)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:58.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.62:41043: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043
01:57:58.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:58.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.61:42857(127.25.36.61:42857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:58.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.36.61:42857: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
01:57:58.875 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043; will retry
01:57:58.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27229), Traces: [0ms] refreshing cache from master, [96ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [286ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [297ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [713ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [722ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [729ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [753ms] refreshing cache from master, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [761ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [773ms] refreshing cache from master, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [779ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [781ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [812ms] refreshing cache from master, [813ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [853ms] refreshing cache from master, [853ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [860ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [861ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [866ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [893ms] refreshing cache from master, [893ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [899ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [903ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [904ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [906ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [953ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [964ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1012ms] refreshing cache from master, [1013ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1233ms] refreshing cache from master, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1240ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1245ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1372ms] refreshing cache from master, [1373ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1382ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1673ms] refreshing cache from master, [1673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1685ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [4113ms] refreshing cache from master, [4113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [4119ms] too many traces: truncated at 100 traces, deferred=Deferred@1768611754(state=PENDING, result=null, callback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155), errback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155)))
01:57:58.886 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.36.60:40959(127.25.36.60:40959)] disconnecting while in state READY
01:57:58.888 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.36.62:41043(127.25.36.62:41043)] disconnecting while in state TERMINATED
01:57:58.888 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.36.61:42857(127.25.36.61:42857)] disconnecting while in state TERMINATED
01:57:58.894 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:57:58.900 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:57:58.901 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.25.36.60:40959(127.25.36.60:40959)] handling channelInactive
01:57:58.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.36.60:40959(127.25.36.60:40959)] cleaning up while in state READY due to: connection closed
01:57:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:57:58.905462 25744 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 26106
01:58:00.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:58:00Z chronyd exiting
01:58:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:58:01.084746 25169 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9b00 after lost signal to thread 25104
01:58:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:58:01.085673 25169 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5140 after lost signal to thread 25172
01:58:01.252 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:58:01.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:58:01.253925 25018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25304
01:58:01.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:58:01.286370 25018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25459
01:58:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:58:01.318352 25018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25603
01:58:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:58:01.350302 25018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25030
01:58:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:58:01.391613 25018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25100
01:58:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:58:01.434187 25018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3pBqiM/build/tsan/bin/kudu with pid 25175
01:58:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:58:02Z chronyd exiting
01:58:03.717 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27229), Traces: [0ms] refreshing cache from master, [96ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [286ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [297ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [713ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [722ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [729ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [753ms] refreshing cache from master, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [761ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [773ms] refreshing cache from master, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [779ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [781ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [812ms] refreshing cache from master, [813ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [853ms] refreshing cache from master, [853ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [860ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [861ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [866ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [893ms] refreshing cache from master, [893ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [899ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [903ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [904ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [906ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [953ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [964ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1012ms] refreshing cache from master, [1013ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1233ms] refreshing cache from master, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1240ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1245ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1372ms] refreshing cache from master, [1373ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1382ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1673ms] refreshing cache from master, [1673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1685ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [4113ms] refreshing cache from master, [4113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [4119ms] too many traces: truncated at 100 traces, deferred=Deferred@1768611754(state=PENDING, result=null, callback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155), errback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.36.61:42857
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27229), Traces: [0ms] refreshing cache from master, [96ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [286ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [297ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [713ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [722ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [729ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [753ms] refreshing cache from master, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [761ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [773ms] refreshing cache from master, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [779ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [781ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [812ms] refreshing cache from master, [813ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [853ms] refreshing cache from master, [853ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [860ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [861ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [866ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [893ms] refreshing cache from master, [893ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [899ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [903ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [904ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [906ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [953ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [964ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1012ms] refreshing cache from master, [1013ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1233ms] refreshing cache from master, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1240ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1245ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1372ms] refreshing cache from master, [1373ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1382ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1673ms] refreshing cache from master, [1673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1685ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [4113ms] refreshing cache from master, [4113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [4119ms] too many traces: truncated at 100 traces, deferred=Deferred@1768611754(state=PENDING, result=null, callback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155), errback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.36.61:42857
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
01:58:04.207 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
01:58:04.207 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27229), Traces: [0ms] refreshing cache from master, [96ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [286ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [297ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [713ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [722ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [729ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [753ms] refreshing cache from master, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [761ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [773ms] refreshing cache from master, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [779ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [781ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [812ms] refreshing cache from master, [813ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [853ms] refreshing cache from master, [853ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [860ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [861ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [866ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [893ms] refreshing cache from master, [893ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [899ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [903ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [904ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [906ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [953ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [964ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1012ms] refreshing cache from master, [1013ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1233ms] refreshing cache from master, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1240ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1245ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1372ms] refreshing cache from master, [1373ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1382ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1673ms] refreshing cache from master, [1673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1685ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [4113ms] refreshing cache from master, [4113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [4119ms] too many traces: truncated at 100 traces, deferred=Deferred@1768611754(state=PENDING, result=null, callback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155), errback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.36.61:42857
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27229), Traces: [0ms] refreshing cache from master, [96ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [286ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [297ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [713ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [722ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [729ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [753ms] refreshing cache from master, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [760ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [761ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [773ms] refreshing cache from master, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [779ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [781ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [812ms] refreshing cache from master, [813ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [853ms] refreshing cache from master, [853ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [860ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [861ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [866ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [893ms] refreshing cache from master, [893ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [899ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [903ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [904ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [906ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [953ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [961ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [964ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1012ms] refreshing cache from master, [1013ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1233ms] refreshing cache from master, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1240ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1245ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1372ms] refreshing cache from master, [1373ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1382ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1673ms] refreshing cache from master, [1673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [1674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [1675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.60:40959: OK, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.62:41043: Network error: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.25.36.61:42857: Network error: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [1685ms] delaying RPC due to: Service unavailable: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857, [4113ms] refreshing cache from master, [4113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.61:42857, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.60:40959, [4115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.36.62:41043, [4119ms] too many traces: truncated at 100 traces, deferred=Deferred@1768611754(state=PENDING, result=null, callback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155), errback=(continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440) -> (continuation of Deferred@1453613094 after retry RPC after error@1127566112) -> (continuation of Deferred@941673762 after retry RPC after error@573530472) -> (continuation of Deferred@1467977819 after retry RPC after error@1048227295) -> (continuation of Deferred@9252591 after retry RPC after error@1508912055) -> (continuation of Deferred@1915131290 after retry RPC after error@425318996) -> (continuation of Deferred@1892316614 after retry RPC after error@683623931) -> (continuation of Deferred@1853071155 after retry RPC after error@959411155) -> (continuation of Deferred@443045237 after retry RPC after error@1219124062) -> (continuation of Deferred@978509695 after retry RPC after error@2023850509) -> (continuation of Deferred@540717969 after retry RPC after error@978349155)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.36.61:42857,127.25.36.60:40959,127.25.36.62:41043) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.62:41043(127.25.36.62:41043): Connection refused: /127.25.36.62:41043,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.36.61:42857(127.25.36.61:42857): Connection refused: /127.25.36.61:42857
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.36.61:42857
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)