01:48:53.956 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:48:53.971 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHBGiCK/test-tmp
01:48:53.974 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:48:53.977 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHBGiCK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:48:54.251 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:48:54.397 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:48:54.398 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:48:54.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.402230 25094 mini_oidc.cc:152] Starting JWKS server
01:48:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.406795 25094 webserver.cc:460] Webserver started at http://127.0.0.1:34771/ using document root <none> and password file <none>
01:48:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.407271 25094 mini_oidc.cc:189] Starting OIDC Discovery server
01:48:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.408987 25094 webserver.cc:460] Webserver started at http://127.0.0.1:33589/ using document root <none> and password file <none>
01:48:54.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:48:54Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:48:54.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:48:54Z Disabled control of system clock
01:48:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.478494 25094 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:48:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:48:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/wal
01:48:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/data
01:48:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:48:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:48:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/logs
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/data/info.pb
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:48:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.190:37309
01:48:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.190
01:48:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:48:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:48:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:48:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:48:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:48:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:48:54.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:48:54.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:48:54.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34771/jwks.json with env {}
01:48:54.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.783018 25106 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:48:54.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.783607 25106 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:48:54.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.784312 25106 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:48:54.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.815150 25106 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:48:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.815443 25106 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:48:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.815641 25106 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:48:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.815817 25106 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:48:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.816069 25106 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:48:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.816277 25106 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34771/jwks.json
01:48:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.851564 25106 master_runner.cc:386] Master server non-default flags:
01:48:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:48:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:48:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:48:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:48:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/data
01:48:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/wal
01:48:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:48:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:48:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:48:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:48:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.190:37309
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34771/jwks.json
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/data/info.pb
01:48:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.190
01:48:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:48:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:48:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:48:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:48:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:48:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:48:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/logs
01:48:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:48:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:48:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:48:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:48:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:48:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:48:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:48:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:48:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:48:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:48:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.852931 25106 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:48:54.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.855245 25106 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:48:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.875522 25113 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:56.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.876197 25115 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:56.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:54.877171 25106 server_base.cc:1034] running on GCE node
01:48:56.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:54.875543 25112 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.072127 25106 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:48:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.075454 25106 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:48:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.076960 25106 hybrid_clock.cc:648] HybridClock initialized: now 1782006536076890 us; error 74 us; skew 500 ppm
01:48:56.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.077931 25106 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:48:56.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.085005 25106 webserver.cc:460] Webserver started at http://127.24.129.190:36377/ using document root <none> and password file <none>
01:48:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.086046 25106 fs_manager.cc:359] Metadata directory not provided
01:48:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.086239 25106 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:48:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.086665 25106 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:48:56.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.091562 25106 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/data/instance:
01:48:56.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "461bfa0e46d249afb180924aef661a09"
01:48:56.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:56 on dist-test-slave-s8kj"
01:48:56.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.092871 25106 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/wal/instance:
01:48:56.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "461bfa0e46d249afb180924aef661a09"
01:48:56.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:56 on dist-test-slave-s8kj"
01:48:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.100858 25106 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
01:48:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.106812 25122 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:48:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.107945 25106 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
01:48:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.108310 25106 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/wal
01:48:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "461bfa0e46d249afb180924aef661a09"
01:48:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:56 on dist-test-slave-s8kj"
01:48:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.108778 25106 fs_report.cc:383] FS layout report
01:48:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:48:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/wal
01:48:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/wal
01:48:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/data/data
01:48:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:48:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:48:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:48:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:48:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:48:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:48:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:48:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:48:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:48:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:48:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:48:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.149768 25106 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:48:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.151412 25106 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:48:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.151999 25106 kserver.cc:163] Server-wide thread pool size limit: 3276
01:48:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.232745 25106 rpc_server.cc:307] RPC server started. Bound to: 127.24.129.190:37309
01:48:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.232839 25173 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.129.190:37309 every 8 connection(s)
01:48:56.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.235860 25106 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/data/info.pb
01:48:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.242367 25174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:48:56.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.246049 25094 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 25106
01:48:56.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.246424 25094 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-0/wal/instance
01:48:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.249827 25094 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:48:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:48:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/wal
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/data
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/logs
01:48:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/data/info.pb
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:48:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:48:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:48:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:48:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.189:33643
01:48:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.189
01:48:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:48:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:48:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:48:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:48:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:48:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:48:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:48:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:48:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34771/jwks.json with env {}
01:48:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.257477 25174 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } has no permanent_uuid. Determining permanent_uuid...
01:48:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.292537 25174 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } has no permanent_uuid. Determining permanent_uuid...
01:48:56.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.295202 25124 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.129.189:33643: connect: Connection refused (error 111)
01:48:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.299494 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.189:33643: Network error: Client connection negotiation failed: client connection to 127.24.129.189:33643: connect: Connection refused (error 111)
01:48:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.349020 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } attempt: 1
01:48:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.354113 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.189:33643: Network error: Client connection negotiation failed: client connection to 127.24.129.189:33643: connect: Connection refused (error 111)
01:48:56.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.422585 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } attempt: 2
01:48:56.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.427460 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.189:33643: Network error: Client connection negotiation failed: client connection to 127.24.129.189:33643: connect: Connection refused (error 111)
01:48:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.518965 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } attempt: 3
01:48:56.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.523523 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.189:33643: Network error: Client connection negotiation failed: client connection to 127.24.129.189:33643: connect: Connection refused (error 111)
01:48:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.576915 25176 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:48:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.577541 25176 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:48:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.578064 25176 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:48:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.609908 25176 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:48:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.610198 25176 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:48:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.610385 25176 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:48:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.610567 25176 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:48:56.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.610740 25176 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:48:56.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.610867 25176 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34771/jwks.json
01:48:56.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.652125 25176 master_runner.cc:386] Master server non-default flags:
01:48:56.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:48:56.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:48:56.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/data
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/wal
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:48:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.189:33643
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34771/jwks.json
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/data/info.pb
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.189
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/logs
01:48:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:48:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:48:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:48:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:48:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:48:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:48:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:48:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:48:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:48:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:48:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.653441 25176 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:48:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.655536 25176 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:48:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.666921 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } attempt: 4
01:48:56.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.671352 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.189:33643: Network error: Client connection negotiation failed: client connection to 127.24.129.189:33643: connect: Connection refused (error 111)
01:48:56.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.676043 25188 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:56.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.970852 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } attempt: 5
01:48:56.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.975391 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.189:33643: Network error: Client connection negotiation failed: client connection to 127.24.129.189:33643: connect: Connection refused (error 111)
01:48:57.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.522816 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } attempt: 6
01:48:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:57.530368 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.189:33643: Network error: Client connection negotiation failed: client connection to 127.24.129.189:33643: connect: Connection refused (error 111)
01:48:57.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.676070 25185 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:56.676076 25186 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:56.676383 25176 server_base.cc:1034] running on GCE node
01:48:57.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.894905 25176 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:48:57.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.898164 25176 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:48:57.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.899646 25176 hybrid_clock.cc:648] HybridClock initialized: now 1782006537899603 us; error 62 us; skew 500 ppm
01:48:57.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.900619 25176 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:48:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.908092 25176 webserver.cc:460] Webserver started at http://127.24.129.189:36929/ using document root <none> and password file <none>
01:48:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.909436 25176 fs_manager.cc:359] Metadata directory not provided
01:48:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.909720 25176 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:48:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.910337 25176 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:48:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.915859 25176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/data/instance:
01:48:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34ae21a6231b42d4a28d294997c9c6af"
01:48:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:57 on dist-test-slave-s8kj"
01:48:57.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.917369 25176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/wal/instance:
01:48:57.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34ae21a6231b42d4a28d294997c9c6af"
01:48:57.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:57 on dist-test-slave-s8kj"
01:48:57.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.926222 25176 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
01:48:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.933028 25197 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:48:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.934378 25176 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
01:48:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.934697 25176 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/wal
01:48:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34ae21a6231b42d4a28d294997c9c6af"
01:48:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:57 on dist-test-slave-s8kj"
01:48:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.935026 25176 fs_report.cc:383] FS layout report
01:48:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:48:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/wal
01:48:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/wal
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/data/data
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:48:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:48:57.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.975642 25176 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:48:57.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.977087 25176 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:48:57.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:57.977489 25176 kserver.cc:163] Server-wide thread pool size limit: 3276
01:48:58.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.058241 25176 rpc_server.cc:307] RPC server started. Bound to: 127.24.129.189:33643
01:48:58.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.058316 25248 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.129.189:33643 every 8 connection(s)
01:48:58.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.061014 25176 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/data/info.pb
01:48:58.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.067803 25249 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:48:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.069957 25094 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 25176
01:48:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.070393 25094 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-1/wal/instance
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.072659 25094 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/wal
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/data
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:48:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/logs
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/data/info.pb
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:48:58.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.188:44799
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.188
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:48:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34771/jwks.json with env {}
01:48:58.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.081480 25249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } has no permanent_uuid. Determining permanent_uuid...
01:48:58.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.110311 25249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } has no permanent_uuid. Determining permanent_uuid...
01:48:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.127877 25249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } has no permanent_uuid. Determining permanent_uuid...
01:48:58.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.130930 25199 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.137722 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.187270 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 1
01:48:58.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.192096 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.260547 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 2
01:48:58.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.264536 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.355973 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 3
01:48:58.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.360311 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.405514 25251 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:48:58.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.406124 25251 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:48:58.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.406831 25251 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:48:58.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.439047 25251 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:48:58.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.439334 25251 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:48:58.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.439518 25251 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:48:58.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.439746 25251 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:48:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.440040 25251 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:48:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.440284 25251 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34771/jwks.json
01:48:58.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.475260 25251 master_runner.cc:386] Master server non-default flags:
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/data
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/wal
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.188:44799
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34771/jwks.json
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/data/info.pb
01:48:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.188
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/logs
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:48:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:48:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:48:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:48:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:48:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:48:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.476640 25251 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:48:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.478667 25251 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:48:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.499150 25264 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:58.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.503844 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 4
01:48:58.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.508368 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.591113 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } attempt: 7
01:48:58.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.610041 25174 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } has no permanent_uuid. Determining permanent_uuid...
01:48:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.617368 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.675823 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 1
01:48:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.680742 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.762152 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 2
01:48:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.767158 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.807916 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 5
01:48:58.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.813671 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.852591 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 3
01:48:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.857937 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:58.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.998389 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 4
01:48:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:59.003345 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:59.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.286944 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 5
01:48:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:59.292387 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.361109 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 6
01:48:59.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:59.368358 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:59.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.499224 25261 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:59.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:58.501909 25251 server_base.cc:1034] running on GCE node
01:48:59.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:58.499336 25262 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:48:59.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.732650 25251 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:48:59.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.735522 25251 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:48:59.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.736943 25251 hybrid_clock.cc:648] HybridClock initialized: now 1782006539736914 us; error 44 us; skew 500 ppm
01:48:59.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.737915 25251 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:48:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.754312 25251 webserver.cc:460] Webserver started at http://127.24.129.188:39497/ using document root <none> and password file <none>
01:48:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.755826 25251 fs_manager.cc:359] Metadata directory not provided
01:48:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.756095 25251 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:48:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.756661 25251 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:48:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.764019 25251 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/data/instance:
01:48:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e41e8c8f119f49bca5064224b64c40ad"
01:48:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:59 on dist-test-slave-s8kj"
01:48:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.765729 25251 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/wal/instance:
01:48:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e41e8c8f119f49bca5064224b64c40ad"
01:48:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:59 on dist-test-slave-s8kj"
01:48:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.776417 25251 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.009s sys 0.000s
01:48:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.785782 25275 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:48:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.787468 25251 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
01:48:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.787973 25251 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/wal
01:48:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e41e8c8f119f49bca5064224b64c40ad"
01:48:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:48:59 on dist-test-slave-s8kj"
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.788484 25251 fs_report.cc:383] FS layout report
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/wal
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/wal
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/data/data
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:48:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:48:59.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:48:59.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:48:59.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:48:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.832448 25251 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:48:59.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.834030 25251 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:48:59.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.834455 25251 kserver.cc:163] Server-wide thread pool size limit: 3276
01:48:59.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.844996 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 6
01:48:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:48:59.855157 25174 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.129.188:44799: Network error: Client connection negotiation failed: client connection to 127.24.129.188:44799: connect: Connection refused (error 111)
01:48:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.929713 25251 rpc_server.cc:307] RPC server started. Bound to: 127.24.129.188:44799
01:48:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.929772 25328 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.129.188:44799 every 8 connection(s)
01:48:59.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.933671 25251 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/data/info.pb
01:48:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.938555 25094 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 25251
01:48:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.938992 25094 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/master-2/wal/instance
01:48:59.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.940147 25329 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:48:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:48:59.969038 25329 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } has no permanent_uuid. Determining permanent_uuid...
01:49:00.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.003125 25329 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } has no permanent_uuid. Determining permanent_uuid...
01:49:00.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.022543 25329 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } has no permanent_uuid. Determining permanent_uuid...
01:49:00.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.043888 25329 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: Bootstrap starting.
01:49:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.050129 25329 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: Neither blocks nor log segments found. Creating new log.
01:49:00.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.052163 25329 log.cc:824] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: Log is configured to *not* fsync() on all Append() calls
01:49:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.059682 25329 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: No bootstrap required, opened a new log
01:49:00.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.093433 25329 raft_consensus.cc:348] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:00.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.094058 25329 raft_consensus.cc:374] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:49:00.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.094280 25329 raft_consensus.cc:729] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e41e8c8f119f49bca5064224b64c40ad, State: Initialized, Role: FOLLOWER
01:49:00.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.094951 25329 consensus_queue.cc:260] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:00.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.097954 25338 sys_catalog.cc:455] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:00.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.098544 25338 sys_catalog.cc:458] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [sys.catalog]: This master's current role is: FOLLOWER
01:49:00.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.098897 25329 sys_catalog.cc:564] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [sys.catalog]: configured and running, proceeding with master startup.
01:49:00.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:00.125962 25349 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:00.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:00.126793 25349 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:00.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.121127 25251 master_runner.cc:186] Error getting master registration for 127.24.129.190:37309: OK, instance_id { permanent_uuid: "461bfa0e46d249afb180924aef661a09" instance_seqno: 1782006536210587 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:49:00.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.147782 25251 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:00.340543 25281 tablet.cc:2369] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:49:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.429020 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 7
01:49:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.450585 25249 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af: Bootstrap starting.
01:49:00.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.456686 25249 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af: Neither blocks nor log segments found. Creating new log.
01:49:00.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.458654 25249 log.cc:824] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af: Log is configured to *not* fsync() on all Append() calls
01:49:00.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.463905 25249 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af: No bootstrap required, opened a new log
01:49:00.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.488684 25249 raft_consensus.cc:348] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:00.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.489424 25249 raft_consensus.cc:374] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:49:00.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.489614 25249 raft_consensus.cc:729] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 34ae21a6231b42d4a28d294997c9c6af, State: Initialized, Role: FOLLOWER
01:49:00.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.490278 25249 consensus_queue.cc:260] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:00.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.493333 25353 sys_catalog.cc:455] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:00.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.494180 25353 sys_catalog.cc:458] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: This master's current role is: FOLLOWER
01:49:00.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.495033 25249 sys_catalog.cc:564] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: configured and running, proceeding with master startup.
01:49:00.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:00.529206 25364 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:00.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:00.530030 25364 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:00.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.525511 25176 master_runner.cc:186] Error getting master registration for 127.24.129.190:37309: OK, instance_id { permanent_uuid: "461bfa0e46d249afb180924aef661a09" instance_seqno: 1782006536210587 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:49:00.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.561625 25176 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:00.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.562136 25353 raft_consensus.cc:482] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:49:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.562736 25353 raft_consensus.cc:504] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.567759 25304 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "34ae21a6231b42d4a28d294997c9c6af" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e41e8c8f119f49bca5064224b64c40ad" is_pre_election: true
01:49:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.568636 25304 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 34ae21a6231b42d4a28d294997c9c6af in term 0.
01:49:00.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.571382 25199 leader_election.cc:302] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34ae21a6231b42d4a28d294997c9c6af, e41e8c8f119f49bca5064224b64c40ad; no voters:
01:49:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.574594 25366 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 0 FOLLOWER]: Leader pre-election won for term 1
01:49:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.575040 25353 leader_election.cc:288] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 461bfa0e46d249afb180924aef661a09 (127.24.129.190:37309), e41e8c8f119f49bca5064224b64c40ad (127.24.129.188:44799)
01:49:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.575235 25366 raft_consensus.cc:482] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:49:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.575590 25366 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 0 FOLLOWER]: Advancing to term 1
01:49:00.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.583057 25366 raft_consensus.cc:504] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:00.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.585528 25366 leader_election.cc:288] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [CANDIDATE]: Term 1 election: Requested vote from peers 461bfa0e46d249afb180924aef661a09 (127.24.129.190:37309), e41e8c8f119f49bca5064224b64c40ad (127.24.129.188:44799)
01:49:00.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.587280 25304 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "34ae21a6231b42d4a28d294997c9c6af" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e41e8c8f119f49bca5064224b64c40ad"
01:49:00.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.587882 25304 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [term 0 FOLLOWER]: Advancing to term 1
01:49:00.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.595371 25304 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 34ae21a6231b42d4a28d294997c9c6af in term 1.
01:49:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.596997 25199 leader_election.cc:302] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34ae21a6231b42d4a28d294997c9c6af, e41e8c8f119f49bca5064224b64c40ad; no voters:
01:49:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.597869 25366 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 1 FOLLOWER]: Leader election won for term 1
01:49:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.598311 25366 raft_consensus.cc:686] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [term 1 LEADER]: Becoming Leader. State: Replica: 34ae21a6231b42d4a28d294997c9c6af, State: Running, Role: LEADER
01:49:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.599633 25366 consensus_queue.cc:237] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:00.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.611313 25353 sys_catalog.cc:455] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: SysCatalogTable state changed. Reason: New leader 34ae21a6231b42d4a28d294997c9c6af. Latest consensus state: current_term: 1 leader_uuid: "34ae21a6231b42d4a28d294997c9c6af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:00.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.612375 25353 sys_catalog.cc:458] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: This master's current role is: LEADER
01:49:00.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.622753 25368 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:49:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.629757 25368 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:49:00.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.654695 25304 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [term 1 FOLLOWER]: Refusing update from remote peer 34ae21a6231b42d4a28d294997c9c6af: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:49:00.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.658679 25353 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [LEADER]: Connected to new peer: Peer: permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:49:00.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.700189 25370 sys_catalog.cc:455] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [sys.catalog]: SysCatalogTable state changed. Reason: New leader 34ae21a6231b42d4a28d294997c9c6af. Latest consensus state: current_term: 1 leader_uuid: "34ae21a6231b42d4a28d294997c9c6af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:00.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.702126 25370 sys_catalog.cc:458] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [sys.catalog]: This master's current role is: FOLLOWER
01:49:00.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.705796 25369 mvcc.cc:205] Tried to move back new op lower bound from 7299098790493351936 to 7299098790307799040. Current Snapshot: MvccSnapshot[applied={T|T < 7299098790493351936}]
01:49:00.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.718175 25353 sys_catalog.cc:455] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "34ae21a6231b42d4a28d294997c9c6af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:00.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.718974 25353 sys_catalog.cc:458] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: This master's current role is: LEADER
01:49:00.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.720618 25368 catalog_manager.cc:1347] Generated new cluster ID: 41256af864cc45b5be61d98e4a1d37bc
01:49:00.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.720901 25368 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:49:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.731523 25370 sys_catalog.cc:455] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "34ae21a6231b42d4a28d294997c9c6af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.733691 25370 sys_catalog.cc:458] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad [sys.catalog]: This master's current role is: FOLLOWER
01:49:00.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.776126 25368 catalog_manager.cc:1370] Generated new certificate authority record
01:49:00.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.787259 25368 catalog_manager.cc:1504] Loading token signing keys...
01:49:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.835935 25368 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af: Generated new TSK 0
01:49:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.837191 25368 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.877669 25094 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/wal
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/data
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/logs
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/data/info.pb
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.129:0
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.129.129
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.129
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:49:00.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.888585 25174 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } attempt: 7
01:49:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:00.968995 25174 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09: Bootstrap starting.
01:49:01.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.013610 25174 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09: Neither blocks nor log segments found. Creating new log.
01:49:01.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.026891 25174 log.cc:824] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09: Log is configured to *not* fsync() on all Append() calls
01:49:01.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.063730 25174 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09: No bootstrap required, opened a new log
01:49:01.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.132989 25349 catalog_manager.cc:1259] Loaded cluster ID: 41256af864cc45b5be61d98e4a1d37bc
01:49:01.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.133678 25349 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: loading cluster ID for follower catalog manager: success
01:49:01.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.142349 25349 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: acquiring CA information for follower catalog manager: success
01:49:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.163210 25349 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e41e8c8f119f49bca5064224b64c40ad: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:49:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.200101 25174 raft_consensus.cc:348] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.201267 25174 raft_consensus.cc:374] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:49:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.202131 25174 raft_consensus.cc:729] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 461bfa0e46d249afb180924aef661a09, State: Initialized, Role: FOLLOWER
01:49:01.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.207307 25174 consensus_queue.cc:260] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } }
01:49:01.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.221107 25378 sys_catalog.cc:455] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:01.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.222021 25378 sys_catalog.cc:458] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [sys.catalog]: This master's current role is: FOLLOWER
01:49:01.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.225102 25174 sys_catalog.cc:564] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [sys.catalog]: configured and running, proceeding with master startup.
01:49:01.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.226055 25147 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [term 0 FOLLOWER]: Advancing to term 1
01:49:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.229126 25149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "34ae21a6231b42d4a28d294997c9c6af" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "461bfa0e46d249afb180924aef661a09" is_pre_election: true
01:49:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.231215 25148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "34ae21a6231b42d4a28d294997c9c6af" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "461bfa0e46d249afb180924aef661a09"
01:49:01.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.236763 25147 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [term 1 FOLLOWER]: Refusing update from remote peer 34ae21a6231b42d4a28d294997c9c6af: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:49:01.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.239071 25353 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [LEADER]: Connected to new peer: Peer: permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:49:01.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.330998 25378 sys_catalog.cc:455] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 34ae21a6231b42d4a28d294997c9c6af. Latest consensus state: current_term: 1 leader_uuid: "34ae21a6231b42d4a28d294997c9c6af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.331158 25386 sys_catalog.cc:455] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "34ae21a6231b42d4a28d294997c9c6af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.331817 25378 sys_catalog.cc:458] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [sys.catalog]: This master's current role is: FOLLOWER
01:49:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.331990 25386 sys_catalog.cc:458] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09 [sys.catalog]: This master's current role is: FOLLOWER
01:49:01.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.369517 25394 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:49:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:01.500401 25375 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:01.501137 25375 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:01.501912 25375 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:01.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:01.557744 25375 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:01.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:01.559140 25375 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.129.129
01:49:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.621713 25375 tablet_server_runner.cc:78] Tablet server non-default flags:
01:49:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:49:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/data
01:49:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/wal
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.129:0
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/data/info.pb
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.129
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:49:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.129.129
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/logs
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.623695 25375 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.625895 25375 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:01.660208 25400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:01.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.717329 25353 sys_catalog.cc:455] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "34ae21a6231b42d4a28d294997c9c6af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "461bfa0e46d249afb180924aef661a09" member_type: VOTER last_known_addr { host: "127.24.129.190" port: 37309 } } peers { permanent_uuid: "34ae21a6231b42d4a28d294997c9c6af" member_type: VOTER last_known_addr { host: "127.24.129.189" port: 33643 } } peers { permanent_uuid: "e41e8c8f119f49bca5064224b64c40ad" member_type: VOTER last_known_addr { host: "127.24.129.188" port: 44799 } } }
01:49:01.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:01.719453 25353 sys_catalog.cc:458] T 00000000000000000000000000000000 P 34ae21a6231b42d4a28d294997c9c6af [sys.catalog]: This master's current role is: LEADER
01:49:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:02.390154 25394 catalog_manager.cc:1259] Loaded cluster ID: 41256af864cc45b5be61d98e4a1d37bc
01:49:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:02.390456 25394 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09: loading cluster ID for follower catalog manager: success
01:49:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:02.419473 25394 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09: acquiring CA information for follower catalog manager: success
01:49:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:02.437536 25394 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 461bfa0e46d249afb180924aef661a09: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:49:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:03.043838 25399 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 25375
01:49:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:04.146131 25399 kernel_stack_watchdog.cc:198] Thread 25375 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 380ms:
01:49:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:49:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:49:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:49:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:49:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:04.147221 25375 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.486s user 0.476s sys 0.836s
01:49:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:04.147588 25375 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.486s user 0.476s sys 0.836s
01:49:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:04.147802 25402 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2488 milliseconds
01:49:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:01.663089 25401 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:04.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:04.149516 25404 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:04.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.149960 25375 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:49:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.153005 25375 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.156239 25375 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:04.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.157688 25375 hybrid_clock.cc:648] HybridClock initialized: now 1782006544157561 us; error 134 us; skew 500 ppm
01:49:04.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.158749 25375 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.167083 25375 webserver.cc:460] Webserver started at http://127.24.129.129:35879/ using document root <none> and password file <none>
01:49:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.168354 25375 fs_manager.cc:359] Metadata directory not provided
01:49:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.168624 25375 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:04.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.169204 25375 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:49:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.176461 25375 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/data/instance:
01:49:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dced344450a41a69ea829be0675bb11"
01:49:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:04 on dist-test-slave-s8kj"
01:49:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.178227 25375 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/wal/instance:
01:49:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dced344450a41a69ea829be0675bb11"
01:49:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:04 on dist-test-slave-s8kj"
01:49:04.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.188265 25375 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
01:49:04.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.196462 25415 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.198355 25375 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.007s sys 0.000s
01:49:04.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.198791 25375 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/wal
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dced344450a41a69ea829be0675bb11"
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:04 on dist-test-slave-s8kj"
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.199245 25375 fs_report.cc:383] FS layout report
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/wal
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/wal
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/data/data
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:04.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.265300 25375 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.267238 25375 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.267786 25375 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:04.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.277278 25375 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:49:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.282306 25375 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:49:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.282538 25375 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:49:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.282728 25375 ts_tablet_manager.cc:610] Registered 0 tablets
01:49:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.282859 25375 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:49:04.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.553126 25375 rpc_server.cc:307] RPC server started. Bound to: 127.24.129.129:32937
01:49:04.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.553246 25528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.129.129:32937 every 8 connection(s)
01:49:04.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.556859 25375 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/data/info.pb
01:49:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.558914 25094 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 25375
01:49:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.559533 25094 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-0/wal/instance
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.575664 25094 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/wal
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/data
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/logs
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/data/info.pb
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.130:0
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.129.130
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.130
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:04.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:49:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.652419 25529 heartbeater.cc:347] Connected to a master server at 127.24.129.188:44799
01:49:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.652957 25529 heartbeater.cc:464] Registering TS with master...
01:49:04.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.654291 25529 heartbeater.cc:511] Master 127.24.129.188:44799 requested a full tablet report, sending...
01:49:04.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.658654 25294 ts_manager.cc:194] Registered new tserver with Master: 2dced344450a41a69ea829be0675bb11 (127.24.129.129:32937)
01:49:04.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.663650 25530 heartbeater.cc:347] Connected to a master server at 127.24.129.189:33643
01:49:04.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.664006 25530 heartbeater.cc:464] Registering TS with master...
01:49:04.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.667135 25530 heartbeater.cc:511] Master 127.24.129.189:33643 requested a full tablet report, sending...
01:49:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.668404 25534 heartbeater.cc:347] Connected to a master server at 127.24.129.190:37309
01:49:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.668749 25534 heartbeater.cc:464] Registering TS with master...
01:49:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.669556 25534 heartbeater.cc:511] Master 127.24.129.190:37309 requested a full tablet report, sending...
01:49:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.671695 25214 ts_manager.cc:194] Registered new tserver with Master: 2dced344450a41a69ea829be0675bb11 (127.24.129.129:32937)
01:49:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.674664 25214 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.129.129:48013
01:49:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:04.679032 25139 ts_manager.cc:194] Registered new tserver with Master: 2dced344450a41a69ea829be0675bb11 (127.24.129.129:32937)
01:49:04.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:04.996074 25533 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:04.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:04.996644 25533 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:04.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:04.997202 25533 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:05.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:05.030530 25533 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:05.031471 25533 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.129.130
01:49:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:05.067497 25533 tablet_server_runner.cc:78] Tablet server non-default flags:
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/data
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/wal
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.130:0
01:49:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/data/info.pb
01:49:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.130
01:49:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.129.130
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/logs
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:05.069033 25533 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:05.071237 25533 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:05.100747 25547 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:05.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:05.679934 25530 heartbeater.cc:503] Master 127.24.129.189:33643 was elected leader, sending a full tablet report...
01:49:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:05.100813 25546 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:05.104018 25549 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:06.346309 25548 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:49:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.346464 25533 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:49:06.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.350631 25533 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.354202 25533 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.355676 25533 hybrid_clock.cc:648] HybridClock initialized: now 1782006546355629 us; error 69 us; skew 500 ppm
01:49:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.356738 25533 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:06.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.367751 25533 webserver.cc:460] Webserver started at http://127.24.129.130:34685/ using document root <none> and password file <none>
01:49:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.368820 25533 fs_manager.cc:359] Metadata directory not provided
01:49:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.369112 25533 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.369575 25533 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:49:06.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.374732 25533 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/data/instance:
01:49:06.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c58b74fb603a473fa27a7fed19cbc504"
01:49:06.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:06 on dist-test-slave-s8kj"
01:49:06.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.375937 25533 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/wal/instance:
01:49:06.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c58b74fb603a473fa27a7fed19cbc504"
01:49:06.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:06 on dist-test-slave-s8kj"
01:49:06.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.383402 25533 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
01:49:06.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.390164 25556 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.391479 25533 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
01:49:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.391971 25533 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/wal
01:49:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c58b74fb603a473fa27a7fed19cbc504"
01:49:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:06 on dist-test-slave-s8kj"
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.392441 25533 fs_report.cc:383] FS layout report
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/wal
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/wal
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/data/data
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:06.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.428468 25533 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.430266 25533 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.430833 25533 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.434167 25533 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:49:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.438694 25533 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:49:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.438913 25533 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:49:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.439204 25533 ts_tablet_manager.cc:610] Registered 0 tablets
01:49:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.439345 25533 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:49:06.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.601438 25533 rpc_server.cc:307] RPC server started. Bound to: 127.24.129.130:42425
01:49:06.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.601536 25669 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.129.130:42425 every 8 connection(s)
01:49:06.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.603873 25533 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/data/info.pb
01:49:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.613726 25094 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 25533
01:49:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.614315 25094 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-1/wal/instance
01:49:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.627247 25094 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/wal
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/data
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/logs
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/data/info.pb
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:49:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.131:0
01:49:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.129.131
01:49:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.131
01:49:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:49:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:49:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:49:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.678581 25671 heartbeater.cc:347] Connected to a master server at 127.24.129.189:33643
01:49:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.679095 25671 heartbeater.cc:464] Registering TS with master...
01:49:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.679890 25670 heartbeater.cc:347] Connected to a master server at 127.24.129.188:44799
01:49:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.680244 25670 heartbeater.cc:464] Registering TS with master...
01:49:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.680294 25671 heartbeater.cc:511] Master 127.24.129.189:33643 requested a full tablet report, sending...
01:49:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.681026 25670 heartbeater.cc:511] Master 127.24.129.188:44799 requested a full tablet report, sending...
01:49:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.683176 25214 ts_manager.cc:194] Registered new tserver with Master: c58b74fb603a473fa27a7fed19cbc504 (127.24.129.130:42425)
01:49:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.683706 25672 heartbeater.cc:347] Connected to a master server at 127.24.129.190:37309
01:49:06.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.684033 25672 heartbeater.cc:464] Registering TS with master...
01:49:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.684894 25672 heartbeater.cc:511] Master 127.24.129.190:37309 requested a full tablet report, sending...
01:49:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.688459 25139 ts_manager.cc:194] Registered new tserver with Master: c58b74fb603a473fa27a7fed19cbc504 (127.24.129.130:42425)
01:49:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.688763 25294 ts_manager.cc:194] Registered new tserver with Master: c58b74fb603a473fa27a7fed19cbc504 (127.24.129.130:42425)
01:49:06.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:06.685227 25214 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.129.130:33739
01:49:06.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:06.994215 25680 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:06.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:06.994701 25680 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:06.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:06.995293 25680 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:07.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:07.027069 25680 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:07.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:07.027915 25680 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.129.131
01:49:07.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:07.068419 25680 tablet_server_runner.cc:78] Tablet server non-default flags:
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.129.148:42979
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/data
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/wal
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.129.131:0
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/data/info.pb
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.129.131
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.129.190:37309,127.24.129.189:33643,127.24.129.188:44799
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.129.131
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/logs
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:07.070057 25680 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:07.072181 25680 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:07.096948 25687 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:07.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:07.694334 25671 heartbeater.cc:503] Master 127.24.129.189:33643 was elected leader, sending a full tablet report...
01:49:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:07.101390 25689 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:07.096954 25686 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:08.426380 25688 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1327 milliseconds
01:49:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.426501 25680 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:49:08.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.428061 25680 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:08.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.430646 25680 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:08.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.432057 25680 hybrid_clock.cc:648] HybridClock initialized: now 1782006548432001 us; error 59 us; skew 500 ppm
01:49:08.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.432946 25680 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:08.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.440202 25680 webserver.cc:460] Webserver started at http://127.24.129.131:46297/ using document root <none> and password file <none>
01:49:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.441370 25680 fs_manager.cc:359] Metadata directory not provided
01:49:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.441651 25680 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.442215 25680 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:49:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.447521 25680 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/data/instance:
01:49:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78451785e5f6489ab76e4b1c204a35ed"
01:49:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:08 on dist-test-slave-s8kj"
01:49:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.448884 25680 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/wal/instance:
01:49:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78451785e5f6489ab76e4b1c204a35ed"
01:49:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:08 on dist-test-slave-s8kj"
01:49:08.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.457022 25680 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
01:49:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.462946 25697 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.464069 25680 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
01:49:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.464468 25680 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/wal
01:49:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78451785e5f6489ab76e4b1c204a35ed"
01:49:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:08 on dist-test-slave-s8kj"
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.464850 25680 fs_report.cc:383] FS layout report
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/wal
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/wal
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/data/data
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:08.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.503516 25680 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:08.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.505187 25680 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:08.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.505728 25680 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:08.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.508719 25680 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:49:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.513953 25680 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:49:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.514218 25680 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:49:08.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.514557 25680 ts_tablet_manager.cc:610] Registered 0 tablets
01:49:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.514757 25680 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:49:08.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.682160 25680 rpc_server.cc:307] RPC server started. Bound to: 127.24.129.131:33117
01:49:08.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.682276 25809 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.129.131:33117 every 8 connection(s)
01:49:08.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.685336 25680 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/data/info.pb
01:49:08.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.693701 25094 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 25680
01:49:08.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.694205 25094 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster5498532384557931522/ts-2/wal/instance
01:49:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.746878 25810 heartbeater.cc:347] Connected to a master server at 127.24.129.188:44799
01:49:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.747380 25810 heartbeater.cc:464] Registering TS with master...
01:49:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.748188 25812 heartbeater.cc:347] Connected to a master server at 127.24.129.190:37309
01:49:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.748207 25811 heartbeater.cc:347] Connected to a master server at 127.24.129.189:33643
01:49:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.748593 25812 heartbeater.cc:464] Registering TS with master...
01:49:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.748597 25811 heartbeater.cc:464] Registering TS with master...
01:49:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.748636 25810 heartbeater.cc:511] Master 127.24.129.188:44799 requested a full tablet report, sending...
01:49:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.749571 25812 heartbeater.cc:511] Master 127.24.129.190:37309 requested a full tablet report, sending...
01:49:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.749678 25811 heartbeater.cc:511] Master 127.24.129.189:33643 requested a full tablet report, sending...
01:49:08.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.752359 25214 ts_manager.cc:194] Registered new tserver with Master: 78451785e5f6489ab76e4b1c204a35ed (127.24.129.131:33117)
01:49:08.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.752671 25294 ts_manager.cc:194] Registered new tserver with Master: 78451785e5f6489ab76e4b1c204a35ed (127.24.129.131:33117)
01:49:08.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.754462 25214 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.129.131:40315
01:49:08.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.754902 25138 ts_manager.cc:194] Registered new tserver with Master: 78451785e5f6489ab76e4b1c204a35ed (127.24.129.131:33117)
01:49:08.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:08.771584 25094 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:49:08.782 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:49:08.784 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:49:08.795 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.129.190"
port: 37309
}
bound_http_address {
host: "127.24.129.190"
port: 36377
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.129.189"
port: 33643
}
bound_http_address {
host: "127.24.129.189"
port: 36929
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.129.188"
port: 44799
}
bound_http_address {
host: "127.24.129.188"
port: 39497
}
}
}
01:49:08.802 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:49:08.805 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.24.129.129"
port: 32937
}
bound_http_address {
host: "127.24.129.129"
port: 35879
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.24.129.130"
port: 42425
}
bound_http_address {
host: "127.24.129.130"
port: 34685
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.24.129.131"
port: 33117
}
bound_http_address {
host: "127.24.129.131"
port: 46297
}
}
}
01:49:08.805 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:49:08.989 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:49:09.106 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHBGiCK/test-tmp
01:49:09.107 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:49:09.107 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHBGiCK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:49:09.110 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:49:09.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:09.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:49:09.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:09.478354 25821 mini_oidc.cc:152] Starting JWKS server
01:49:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:09.482012 25821 webserver.cc:460] Webserver started at http://127.0.0.1:39517/ using document root <none> and password file <none>
01:49:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:09.482538 25821 mini_oidc.cc:189] Starting OIDC Discovery server
01:49:09.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:09.484666 25821 webserver.cc:460] Webserver started at http://127.0.0.1:44473/ using document root <none> and password file <none>
01:49:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:49:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/principal' for realm 'KRBTEST.COM',
01:49:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:49:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:09 dist-test-slave-s8kj krb5kdc[25833](info): setting up network...
01:49:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:49:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:09 dist-test-slave-s8kj krb5kdc[25833](info): set up 2 sockets
01:49:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:09 dist-test-slave-s8kj krb5kdc[25833](info): commencing operation
01:49:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:49:09.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:09.758697 25811 heartbeater.cc:503] Master 127.24.129.189:33643 was elected leader, sending a full tablet report...
01:49:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:10.603150 25821 mini_kdc.cc:121] Time spent starting KDC: real 1.091s user 0.000s sys 0.021s
01:49:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:49:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:49:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:49:10.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:49:10.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:49:10.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:49:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:49:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:49:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:49:10.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:49:10.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/test-user.keytab.
01:49:10.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/test-user.keytab.
01:49:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:10 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006550, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:49:10.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:49:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:49:10.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:49:10Z Disabled control of system clock
01:49:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.55.126@KRBTEST.COM; defaulting to no policy
01:49:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:49:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.55.126@KRBTEST.COM" created.
01:49:10.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:49:10.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.55.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.126.keytab.
01:49:10.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.55.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.126.keytab.
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:10.814857 25821 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/logs
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/info.pb
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.126:41357
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.126
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.126.keytab
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.126
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:49:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:49:11.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.132995 25846 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:11.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.133741 25846 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.134450 25846 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:11.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.166831 25846 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:49:11.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.167110 25846 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:49:11.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.167484 25846 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:11.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.167811 25846 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:49:11.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.168108 25846 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:49:11.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.168404 25846 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:49:11.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.168640 25846 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39517/jwks.json
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:11.206835 25846 master_runner.cc:386] Master server non-default flags:
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.126.keytab
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.126
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.126:41357
01:49:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/info.pb
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.126
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/logs
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:11.208365 25846 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:11.209656 25846 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.231727 25853 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.233031 25852 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:11.231909 25855 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:12.493249 25854 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:49:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.493556 25846 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:49:12.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.498345 25846 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.501269 25846 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.502668 25846 hybrid_clock.cc:648] HybridClock initialized: now 1782006552502638 us; error 76 us; skew 500 ppm
01:49:12.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.503587 25846 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:12.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:12 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006552, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:12.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.509896 25846 init.cc:377] Logged in from keytab as kudu/127.25.55.126@KRBTEST.COM (short username kudu)
01:49:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.515277 25846 webserver.cc:460] Webserver started at http://127.25.55.126:41355/ using document root <none> and password file <none>
01:49:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.516163 25846 fs_manager.cc:359] Metadata directory not provided
01:49:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.516347 25846 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.516722 25846 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:49:12.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.521577 25846 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/instance:
01:49:12.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8f3c5485ccb4490a2ea7dad5870c3af"
01:49:12.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:12 on dist-test-slave-s8kj"
01:49:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.523392 25846 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal/instance:
01:49:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8f3c5485ccb4490a2ea7dad5870c3af"
01:49:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:12 on dist-test-slave-s8kj"
01:49:12.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.531364 25846 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
01:49:12.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.538303 25864 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:12.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.539520 25846 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.539880 25846 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8f3c5485ccb4490a2ea7dad5870c3af"
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:12 on dist-test-slave-s8kj"
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.540199 25846 fs_report.cc:383] FS layout report
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/data
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:12.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:12.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:12.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:12.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.589362 25846 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.595799 25846 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.596303 25846 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.675911 25846 rpc_server.cc:307] RPC server started. Bound to: 127.25.55.126:41357
01:49:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.676000 25915 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.55.126:41357 every 8 connection(s)
01:49:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.678490 25846 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/info.pb
01:49:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.682983 25821 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 25846
01:49:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.683537 25821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal/instance
01:49:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.685184 25916 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:49:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.701713 25916 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } has no permanent_uuid. Determining permanent_uuid...
01:49:12.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.55.125@KRBTEST.COM; defaulting to no policy
01:49:12.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:49:12.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.55.125@KRBTEST.COM" created.
01:49:12.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:12 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006552, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.126@KRBTEST.COM for kudu/127.25.55.126@KRBTEST.COM
01:49:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:49:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.55.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.125.keytab.
01:49:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.55.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.125.keytab.
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.746050 25821 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/logs
01:49:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/info.pb
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.125:37313
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.125
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.125.keytab
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.125
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:49:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.752550 25916 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } has no permanent_uuid. Determining permanent_uuid...
01:49:12.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:12.755864 25866 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:12.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:12.760393 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.125:37313: Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.809976 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } attempt: 1
01:49:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:12.813892 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.125:37313: Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.882355 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } attempt: 2
01:49:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:12.886933 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.125:37313: Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:12.978387 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } attempt: 3
01:49:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:12.982658 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.125:37313: Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.075770 25922 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.076404 25922 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:13.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.076982 25922 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:13.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.109102 25922 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:49:13.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.109437 25922 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:49:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.109802 25922 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.110015 25922 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:49:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.110293 25922 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:49:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.110584 25922 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:49:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.110813 25922 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39517/jwks.json
01:49:13.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:13.126117 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } attempt: 4
01:49:13.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.130505 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.125:37313: Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:13.150996 25922 master_runner.cc:386] Master server non-default flags:
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.125.keytab
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.125
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.125:37313
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/info.pb
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.125
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/logs
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:13.152489 25922 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:13.153861 25922 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:13.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.176438 25930 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:13.429914 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } attempt: 5
01:49:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.434639 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.125:37313: Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:13.982272 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } attempt: 6
01:49:13.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.991156 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.125:37313: Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.177402 25929 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:13.177711 25932 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:14.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:13.179128 25922 server_base.cc:1034] running on GCE node
01:49:14.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.451753 25922 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:14.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.454402 25922 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:14.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.455821 25922 hybrid_clock.cc:648] HybridClock initialized: now 1782006554455763 us; error 66 us; skew 500 ppm
01:49:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.456808 25922 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:14.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:14 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006554, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:14.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.468724 25922 init.cc:377] Logged in from keytab as kudu/127.25.55.125@KRBTEST.COM (short username kudu)
01:49:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.474197 25922 webserver.cc:460] Webserver started at http://127.25.55.125:33987/ using document root <none> and password file <none>
01:49:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.475114 25922 fs_manager.cc:359] Metadata directory not provided
01:49:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.475301 25922 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.475683 25922 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:49:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.480536 25922 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/instance:
01:49:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1949606339324dc58c371835ff561fdc"
01:49:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:14 on dist-test-slave-s8kj"
01:49:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.481684 25922 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal/instance:
01:49:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1949606339324dc58c371835ff561fdc"
01:49:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:14 on dist-test-slave-s8kj"
01:49:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.489382 25922 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
01:49:14.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.495555 25943 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.496861 25922 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
01:49:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.497251 25922 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1949606339324dc58c371835ff561fdc"
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:14 on dist-test-slave-s8kj"
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.497637 25922 fs_report.cc:383] FS layout report
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/data
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:14.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.553807 25922 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.560089 25922 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.560622 25922 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.658963 25922 rpc_server.cc:307] RPC server started. Bound to: 127.25.55.125:37313
01:49:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.659078 25994 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.55.125:37313 every 8 connection(s)
01:49:14.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.662034 25922 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/info.pb
01:49:14.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.667462 25821 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 25922
01:49:14.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.667575 25995 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:49:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.667879 25821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal/instance
01:49:14.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.680449 25995 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } has no permanent_uuid. Determining permanent_uuid...
01:49:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.55.124@KRBTEST.COM; defaulting to no policy
01:49:14.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:49:14.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.55.124@KRBTEST.COM" created.
01:49:14.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:14 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006554, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.125@KRBTEST.COM for kudu/127.25.55.126@KRBTEST.COM
01:49:14.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:49:14.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.55.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.124.keytab.
01:49:14.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.55.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.124.keytab.
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.726538 25821 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/logs
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/info.pb
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:49:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.124:42341
01:49:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.124
01:49:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.124.keytab
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.124
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:49:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.734241 25995 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } has no permanent_uuid. Determining permanent_uuid...
01:49:14.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:14 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006554, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.125@KRBTEST.COM for kudu/127.25.55.125@KRBTEST.COM
01:49:14.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.774089 25995 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } has no permanent_uuid. Determining permanent_uuid...
01:49:14.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:14.777110 25946 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:14.785322 25995 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.834964 25995 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 1
01:49:14.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:14.839298 25995 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:14.907761 25995 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 2
01:49:14.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:14.912335 25995 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.003790 25995 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 3
01:49:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.007364 25995 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.051618 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } attempt: 7
01:49:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:15 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006552, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.126@KRBTEST.COM for kudu/127.25.55.125@KRBTEST.COM
01:49:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.074597 25916 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } has no permanent_uuid. Determining permanent_uuid...
01:49:15.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.080068 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.093948 26001 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:15.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.094585 26001 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.095286 26001 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:15.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.127367 26001 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:49:15.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.127662 26001 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:49:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.127929 26001 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.128124 26001 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:49:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.128295 26001 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:49:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.128475 26001 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:49:15.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.128614 26001 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39517/jwks.json
01:49:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.138523 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 1
01:49:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.142339 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.150839 25995 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 4
01:49:15.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.154827 25995 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.168763 26001 master_runner.cc:386] Master server non-default flags:
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.124.keytab
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.124
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.124:42341
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:49:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/info.pb
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.124
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/logs
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.170189 26001 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.171363 26001 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.189412 26014 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.223788 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 2
01:49:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.228839 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.314323 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 3
01:49:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.319334 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.454285 25995 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 5
01:49:15.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.457949 25995 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.459833 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 4
01:49:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.464532 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:15.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.748154 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 5
01:49:15.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.753116 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.005587 25995 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 6
01:49:16.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:16.012832 25995 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:16.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.305581 25916 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } attempt: 6
01:49:16.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:16.313640 25916 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.55.124:42341: Network error: Client connection negotiation failed: client connection to 127.25.55.124:42341: connect: Connection refused (error 111)
01:49:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.189481 26013 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:15.191421 26001 server_base.cc:1034] running on GCE node
01:49:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:15.189754 26016 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:16.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.465955 26001 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.468704 26001 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.470103 26001 hybrid_clock.cc:648] HybridClock initialized: now 1782006556470079 us; error 108 us; skew 500 ppm
01:49:16.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.470917 26001 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:16.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006556, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:16.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.477444 26001 init.cc:377] Logged in from keytab as kudu/127.25.55.124@KRBTEST.COM (short username kudu)
01:49:16.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.482921 26001 webserver.cc:460] Webserver started at http://127.25.55.124:34967/ using document root <none> and password file <none>
01:49:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.483814 26001 fs_manager.cc:359] Metadata directory not provided
01:49:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.484023 26001 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.484491 26001 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:49:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.489535 26001 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/instance:
01:49:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7464af44d83e4841b6113e1c7032c358"
01:49:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:16 on dist-test-slave-s8kj"
01:49:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.490597 26001 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal/instance:
01:49:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7464af44d83e4841b6113e1c7032c358"
01:49:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:16 on dist-test-slave-s8kj"
01:49:16.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.498278 26001 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
01:49:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.504611 26028 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:16.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.506197 26001 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
01:49:16.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.506520 26001 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7464af44d83e4841b6113e1c7032c358"
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:16 on dist-test-slave-s8kj"
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.506851 26001 fs_report.cc:383] FS layout report
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/data
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:16.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:16.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.541014 26001 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:16.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.547103 26001 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:16.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.547605 26001 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.638374 26001 rpc_server.cc:307] RPC server started. Bound to: 127.25.55.124:42341
01:49:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.638437 26080 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.55.124:42341 every 8 connection(s)
01:49:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.641136 26001 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/info.pb
01:49:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.646942 25821 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 26001
01:49:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.647341 26081 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:49:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.647500 25821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal/instance
01:49:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.665093 26081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } has no permanent_uuid. Determining permanent_uuid...
01:49:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006550, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.55.124@KRBTEST.COM
01:49:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006556, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for kudu/127.25.55.126@KRBTEST.COM
01:49:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.728579 26081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } has no permanent_uuid. Determining permanent_uuid...
01:49:16.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006550, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.55.125@KRBTEST.COM
01:49:16.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006556, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for kudu/127.25.55.125@KRBTEST.COM
01:49:16.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.760257 26081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } has no permanent_uuid. Determining permanent_uuid...
01:49:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006550, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.55.126@KRBTEST.COM
01:49:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006556, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for kudu/127.25.55.124@KRBTEST.COM
01:49:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.791646 25821 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:49:16.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.795878 26081 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: Bootstrap starting.
01:49:16.799 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:16.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:49:16.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.802678 26081 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: Neither blocks nor log segments found. Creating new log.
01:49:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.805680 26081 log.cc:824] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: Log is configured to *not* fsync() on all Append() calls
01:49:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.811826 26081 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: No bootstrap required, opened a new log
01:49:16.816 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:49:16.817 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:49:16.821 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.55.126"
port: 41357
}
bound_http_address {
host: "127.25.55.126"
port: 41355
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.55.125"
port: 37313
}
bound_http_address {
host: "127.25.55.125"
port: 33987
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.55.124"
port: 42341
}
bound_http_address {
host: "127.25.55.124"
port: 34967
}
}
}
01:49:16.822 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:49:16.822 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:16.822 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:49:16.824 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:49:16.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.841977 26081 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:16.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.842610 26081 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:49:16.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.842792 26081 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7464af44d83e4841b6113e1c7032c358, State: Initialized, Role: FOLLOWER
01:49:16.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.843532 26081 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.846544 26092 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } } }
01:49:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.847460 26092 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [sys.catalog]: This master's current role is: FOLLOWER
01:49:16.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.848744 26081 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [sys.catalog]: configured and running, proceeding with master startup.
01:49:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:49:16.867 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:16.869 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc
01:49:16.875 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:49:16.877 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc
01:49:16.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:16.880682 26103 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:16.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:16.881040 26103 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:16.881 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:49:16.885 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
01:49:16.887 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:49:16.887 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:49:16.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.876389 26001 master_runner.cc:186] Error getting master registration for 127.25.55.126:41357: OK, instance_id { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" instance_seqno: 1782006552653792 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:49:16.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.914875 26001 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:16.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:16 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:49:16.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:16.930 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc
01:49:16.932 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:49:16.933 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.55.126:41357
01:49:16.935 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:49:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.936331 25821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 25846
01:49:16.970 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:16.971 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.55.124:42341
01:49:16.971 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:49:16.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:16.973708 25821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 26001
01:49:17.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:17.000 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.55.125:37313
01:49:17.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:49:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:17.000967 25821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 25922
01:49:17.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:17.023 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.55.126:41357
01:49:17.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:17.027254 25821 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/logs
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/info.pb
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.126:41357
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.126
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41355
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.126.keytab
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.126
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:49:17.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.359867 26108 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:17.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.360559 26108 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:17.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.361351 26108 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.393842 26108 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:49:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.394158 26108 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:49:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.394434 26108 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.394639 26108 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:49:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.394825 26108 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:49:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.395190 26108 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:49:17.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.395370 26108 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39517/jwks.json
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:17.435006 26108 master_runner.cc:386] Master server non-default flags:
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.126.keytab
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.126
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.126:41357
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/info.pb
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.126
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41355
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/logs
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:17.436201 26108 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:17.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:17.437311 26108 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.456401 26117 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.456420 26115 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:17.459316 26108 server_base.cc:1034] running on GCE node
01:49:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:17.457129 26114 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:18.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.718415 26108 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:18.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.721736 26108 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:18.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.723218 26108 hybrid_clock.cc:648] HybridClock initialized: now 1782006558723171 us; error 66 us; skew 500 ppm
01:49:18.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.724225 26108 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:18.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:18 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006558, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:18.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.730855 26108 init.cc:377] Logged in from keytab as kudu/127.25.55.126@KRBTEST.COM (short username kudu)
01:49:18.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.736608 26108 webserver.cc:460] Webserver started at http://127.25.55.126:41355/ using document root <none> and password file <none>
01:49:18.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.737709 26108 fs_manager.cc:359] Metadata directory not provided
01:49:18.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.738024 26108 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:18.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.747038 26108 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
01:49:18.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.751785 26126 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:18.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.753230 26108 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
01:49:18.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.753646 26108 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:18.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8f3c5485ccb4490a2ea7dad5870c3af"
01:49:18.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:12 on dist-test-slave-s8kj"
01:49:18.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.757927 26108 fs_report.cc:383] FS layout report
01:49:18.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:18.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:18.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/data
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.811558 26108 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.817276 26108 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.817673 26108 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:18.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.888541 26108 rpc_server.cc:307] RPC server started. Bound to: 127.25.55.126:41357
01:49:18.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.888659 26177 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.55.126:41357 every 8 connection(s)
01:49:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.891526 26108 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/data/info.pb
01:49:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.896436 25821 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 26108
01:49:18.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:18.897 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.55.124:42341
01:49:18.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.898751 25821 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/logs
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/info.pb
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:49:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.124:42341
01:49:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.124
01:49:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34967
01:49:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.124.keytab
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.124
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:49:18.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.903668 26178 sys_catalog.cc:263] Verifying existing consensus state
01:49:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 01:49:18.904534 26178 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:49:18.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.905015 26108 master.cc:537] Master@127.25.55.126:41357 shutting down...
01:49:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:18.986245 26108 master.cc:552] Master@127.25.55.126:41357 shutdown complete.
01:49:18.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:49:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.230406 26180 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:19.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.231014 26180 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:19.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.231580 26180 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.263705 26180 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:49:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.263929 26180 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:49:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.264259 26180 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.264554 26180 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:49:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.264868 26180 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:49:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.265202 26180 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:49:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.265452 26180 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39517/jwks.json
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:19.308723 26180 master_runner.cc:386] Master server non-default flags:
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.124.keytab
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.124
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.124:42341
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/info.pb
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.124
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34967
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/logs
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:19.310096 26180 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:19.311342 26180 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:19.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.329877 26189 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:19.331717 26180 server_base.cc:1034] running on GCE node
01:49:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.329851 26191 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:19.329865 26188 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:20.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.613482 26180 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.616178 26180 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.617640 26180 hybrid_clock.cc:648] HybridClock initialized: now 1782006560617608 us; error 43 us; skew 500 ppm
01:49:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.618635 26180 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:20.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:20 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006560, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:20.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.624917 26180 init.cc:377] Logged in from keytab as kudu/127.25.55.124@KRBTEST.COM (short username kudu)
01:49:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.630113 26180 webserver.cc:460] Webserver started at http://127.25.55.124:34967/ using document root <none> and password file <none>
01:49:20.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.631168 26180 fs_manager.cc:359] Metadata directory not provided
01:49:20.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.631374 26180 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.639786 26180 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.002s
01:49:20.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.644541 26200 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.646011 26180 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:49:20.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.646324 26180 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:20.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7464af44d83e4841b6113e1c7032c358"
01:49:20.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:16 on dist-test-slave-s8kj"
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.648727 26180 fs_report.cc:383] FS layout report
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/wal
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/data
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.696601 26180 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:20.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.703250 26180 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:20.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.703685 26180 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.780509 26180 rpc_server.cc:307] RPC server started. Bound to: 127.25.55.124:42341
01:49:20.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.780627 26251 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.55.124:42341 every 8 connection(s)
01:49:20.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.783447 26180 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-2/data/info.pb
01:49:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.785884 25821 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 26180
01:49:20.787 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:20.787 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.55.125:37313
01:49:20.788 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:49:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.789603 25821 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/logs
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/info.pb
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.125:37313
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.125
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33987
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.125.keytab
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.125
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:49:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.795410 26252 sys_catalog.cc:263] Verifying existing consensus state
01:49:20.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.801823 26252 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: Bootstrap starting.
01:49:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.847116 26252 log.cc:824] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: Log is configured to *not* fsync() on all Append() calls
01:49:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.852206 26252 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:49:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.853108 26252 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: Bootstrap complete.
01:49:20.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.888890 26252 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:20.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.892421 26252 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:49:20.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.892799 26252 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7464af44d83e4841b6113e1c7032c358, State: Initialized, Role: FOLLOWER
01:49:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.893793 26252 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.896775 26256 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } } }
01:49:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.897565 26256 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [sys.catalog]: This master's current role is: FOLLOWER
01:49:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.898717 26252 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [sys.catalog]: configured and running, proceeding with master startup.
01:49:20.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:20.934623 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:20.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:20.934912 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:20.959555 26206 tablet.cc:2369] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:49:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.933854 26180 master_runner.cc:186] Error getting master registration for 127.25.55.126:41357: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111),
01:49:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:20.969709 26180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:21.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:21.095237 26256 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:49:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:21.095824 26256 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:21.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:21.099148 26256 leader_election.cc:288] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357), 1949606339324dc58c371835ff561fdc (127.25.55.125:37313)
01:49:21.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.100615 26204 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111)
01:49:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.106745 26204 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357): Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111)
01:49:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.108554 26202 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1949606339324dc58c371835ff561fdc (127.25.55.125:37313): Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:21.109043 26202 leader_election.cc:302] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7464af44d83e4841b6113e1c7032c358; no voters: 1949606339324dc58c371835ff561fdc, d8f3c5485ccb4490a2ea7dad5870c3af
01:49:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:21.110157 26256 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:49:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.181775 26254 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:49:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.182509 26254 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:49:21.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.183444 26254 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:49:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.218627 26254 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:49:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.218919 26254 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:49:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.219312 26254 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:49:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.219638 26254 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:49:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.219941 26254 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:49:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.220254 26254 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:49:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.220480 26254 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39517/jwks.json
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:21.260844 26254 master_runner.cc:386] Master server non-default flags:
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.55.84:38669
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.55.126:41357,127.25.55.125:37313,127.25.55.124:42341
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/krb5kdc/kudu_127.25.55.125.keytab
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.55.125
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.55.125:37313
01:49:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39517/jwks.json
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/info.pb
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.55.125
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33987
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/logs
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:49:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:21.262401 26254 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:49:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:21.263583 26254 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:49:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.284070 26277 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.941449 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.942368 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.287804 26278 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:22.558857 26279 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:49:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.559384 26254 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:49:22.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:21.286078 26280 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:49:22.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.566124 26254 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:49:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.569615 26254 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:49:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.571333 26254 hybrid_clock.cc:648] HybridClock initialized: now 1782006562571276 us; error 57 us; skew 500 ppm
01:49:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.572376 26254 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:49:22.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:22 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006562, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.583437 26254 init.cc:377] Logged in from keytab as kudu/127.25.55.125@KRBTEST.COM (short username kudu)
01:49:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.589720 26254 webserver.cc:460] Webserver started at http://127.25.55.125:33987/ using document root <none> and password file <none>
01:49:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.590736 26254 fs_manager.cc:359] Metadata directory not provided
01:49:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.591001 26254 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:49:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.599905 26254 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
01:49:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.616356 26289 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:49:22.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.617998 26254 fs_manager.cc:725] Time spent opening block manager: real 0.016s user 0.001s sys 0.010s
01:49:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.618404 26254 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data,/tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1949606339324dc58c371835ff561fdc"
01:49:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:49:14 on dist-test-slave-s8kj"
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.621747 26254 fs_report.cc:383] FS layout report
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/data
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:49:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:49:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:49:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.668784 26254 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:49:22.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.676436 26254 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:49:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.677014 26254 kserver.cc:163] Server-wide thread pool size limit: 3276
01:49:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.764775 26254 rpc_server.cc:307] RPC server started. Bound to: 127.25.55.125:37313
01:49:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.764963 26340 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.55.125:37313 every 8 connection(s)
01:49:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.767956 26254 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/data/info.pb
01:49:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.768682 25821 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu as pid 26254
01:49:22.769 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:49:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.803434 26341 sys_catalog.cc:263] Verifying existing consensus state
01:49:22.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 01:49:22.807629 26341 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:49:22.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.808151 26254 master.cc:537] Master@127.25.55.125:37313 shutting down...
01:49:22.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.810423 26344 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:49:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.814157 26344 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.816869 26344 leader_election.cc:288] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357), 1949606339324dc58c371835ff561fdc (127.25.55.125:37313)
01:49:22.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:22.839489 26204 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357): Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111)
01:49:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:22 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006560, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for kudu/127.25.55.125@KRBTEST.COM
01:49:22.865 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:22.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:22.878208 26293 connection.cc:441] server connection from 127.0.0.1:59268 torn down before Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:59268 (request call id 0) could send its response
01:49:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:22.881515 26202 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1949606339324dc58c371835ff561fdc (127.25.55.125:37313): Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.881956 26202 leader_election.cc:302] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7464af44d83e4841b6113e1c7032c358; no voters: 1949606339324dc58c371835ff561fdc, d8f3c5485ccb4490a2ea7dad5870c3af
01:49:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.883210 26344 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:49:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:22.940830 26254 master.cc:552] Master@127.25.55.125:37313 shutdown complete.
01:49:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:22.946739 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:22.947728 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHBGiCK/test-tmp/mini-kudu-cluster502486968318090852/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:49:22.969 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.55.124:42341(127.25.55.124:42341)] connecting to peer
01:49:22.977 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.015 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.25.55.124:42341(127.25.55.124:42341)] Successfully connected to peer
01:49:23.017 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.020 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.021 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.021 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.022 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.096 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
01:49:23.104 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.25.55.124
01:49:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:23 dist-test-slave-s8kj krb5kdc[25833](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.55.124@KRBTEST.COM
01:49:23.321 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8cde05a8, L:/127.0.0.1:51170 - R:/127.25.55.124:42341] using SASL/GSSAPI
01:49:23.337 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.347 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.352 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.354 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.354 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.355 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.355 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.355 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.356 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.358 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.368 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.370 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.374 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.375 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.375 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.376 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.376 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.387 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.389 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.395 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.397 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.407 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.409 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.415 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.449 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.455 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.527 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.530 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.607 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.615 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.667 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.807 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:23.809 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:23.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:23.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:23.814 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:23.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:23.951022 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:23.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:23.951310 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:24.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:24.048 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:24.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:24.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:24.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:24.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:24.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:24.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:24.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:24.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:24.736507 26353 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:49:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:24.736892 26353 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:24.738476 26353 leader_election.cc:288] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357), 1949606339324dc58c371835ff561fdc (127.25.55.125:37313)
01:49:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:24.743798 26204 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357): Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111)
01:49:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:24.744254 26202 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1949606339324dc58c371835ff561fdc (127.25.55.125:37313): Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:24.744649 26202 leader_election.cc:302] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7464af44d83e4841b6113e1c7032c358; no voters: 1949606339324dc58c371835ff561fdc, d8f3c5485ccb4490a2ea7dad5870c3af
01:49:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:24.745285 26353 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:49:24.907 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:24.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:24.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:24.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:24.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:24.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:24.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:24.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:24.913 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:24.954460 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:24.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:24.954777 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:25.958636 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:25.958976 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:25.972705 26180 master_runner.cc:186] Error getting master registration for 127.25.55.126:41357: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111),
01:49:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:25.973137 26180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:26.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:26.948839 25525 debug-util.cc:398] Leaking SignalData structure 0x7b08000bef60 after lost signal to thread 25395
01:49:26.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:26.950044 25525 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb600 after lost signal to thread 25528
01:49:26.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:26.962229 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:26.962529 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:27.727 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:27.730 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:27.730 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:27.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:27.733 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:27.734 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:27.734 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:27.735 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:27.735 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:27.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:27.965528 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:27.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:27.965799 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:28.783540 26364 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:49:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:28.783972 26364 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:28.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:28.785755 26364 leader_election.cc:288] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357), 1949606339324dc58c371835ff561fdc (127.25.55.125:37313)
01:49:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:28.788039 26204 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111) [suppressed 5 similar messages]
01:49:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:28.790846 26202 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1949606339324dc58c371835ff561fdc (127.25.55.125:37313): Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:28.791304 26204 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357): Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111)
01:49:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:28.791618 26204 leader_election.cc:302] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7464af44d83e4841b6113e1c7032c358; no voters: 1949606339324dc58c371835ff561fdc, d8f3c5485ccb4490a2ea7dad5870c3af
01:49:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:28.792119 26364 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:49:28.927 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:28.928 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:28.930 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:28.930 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:28.930 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:28.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:28.931 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:28.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:28.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:28.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:28.968868 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:28.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:28.969249 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:29.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:29.973301 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:29.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:29.973760 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:30 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006570, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:30.629688 26196 init.cc:295] Successfully reacquired a new kerberos TGT
01:49:30.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:30.977025 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:30.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:30.977025 26180 master_runner.cc:186] Error getting master registration for 127.25.55.126:41357: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111),
01:49:30.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:30.977499 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:30.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:30.977716 26180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:31.980729 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:31.981010 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:32.247 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:32.249 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:32.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:32.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:32.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:32.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:32.253 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:32.253 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:32.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:32.890166 26372 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:49:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:32.890610 26372 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:32.892234 26372 leader_election.cc:288] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357), 1949606339324dc58c371835ff561fdc (127.25.55.125:37313)
01:49:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:32.896989 26204 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357): Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111)
01:49:32.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:32.897702 26202 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1949606339324dc58c371835ff561fdc (127.25.55.125:37313): Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:32.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:32.898031 26202 leader_election.cc:302] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7464af44d83e4841b6113e1c7032c358; no voters: 1949606339324dc58c371835ff561fdc, d8f3c5485ccb4490a2ea7dad5870c3af
01:49:32.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:32.898741 26372 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:49:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:32.983937 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:32.984202 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:33.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:33.987529 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:33.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:33.987859 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:34.279714 25245 debug-util.cc:398] Leaking SignalData structure 0x7b080008fe40 after lost signal to thread 25180
01:49:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:34.280488 25245 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc3a0 after lost signal to thread 25248
01:49:34.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:34.990898 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:34.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:34.991199 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:35.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:35.980679 26180 master_runner.cc:186] Error getting master registration for 127.25.55.126:41357: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111),
01:49:35.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:35.981138 26180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:35.994496 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:35.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:35.994855 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:36.087 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:36.089 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:36.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:36.091 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:36.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:36.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:36.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:36.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:36.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:36.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:36.236934 25525 debug-util.cc:398] Leaking SignalData structure 0x7b08000af840 after lost signal to thread 25395
01:49:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:36.237929 25525 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb480 after lost signal to thread 25528
01:49:36.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:36.997884 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:36.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:36.998240 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:37.506314 26382 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:49:37.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:37.506734 26382 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:37.508685 26382 leader_election.cc:288] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357), 1949606339324dc58c371835ff561fdc (127.25.55.125:37313)
01:49:37.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:37.510946 26204 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111) [suppressed 3 similar messages]
01:49:37.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:37.513600 26202 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1949606339324dc58c371835ff561fdc (127.25.55.125:37313): Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:37.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:37.514019 26204 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357): Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111)
01:49:37.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:37.514299 26204 leader_election.cc:302] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7464af44d83e4841b6113e1c7032c358; no voters: 1949606339324dc58c371835ff561fdc, d8f3c5485ccb4490a2ea7dad5870c3af
01:49:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:37.514847 26382 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:49:37.607 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:37.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:37.610 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:37.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:37.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:37.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:37.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:37.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:37.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:38.001657 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:38.001956 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:38.707 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:38.709 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:38.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:38.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:38.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:38.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:38.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:38.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:38.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:39.004923 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:39.005247 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:40.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:40.009294 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:40.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:40.009773 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:40 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006580, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:40.632416 26196 init.cc:295] Successfully reacquired a new kerberos TGT
01:49:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:40.984855 26180 master_runner.cc:186] Error getting master registration for 127.25.55.126:41357: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111),
01:49:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:40.985440 26180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:41.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:41.012687 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:41.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:41.013016 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:41.267 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:41.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:41.270 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:41.270 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:41.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:41.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:41.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:41.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:41.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:42.015782 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:42.016076 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:42.167 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:42.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:42.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:42.170 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:42.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:42.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:42.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:42.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:42.172 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:43.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:43.019063 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:43.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:43.019446 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:44.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:44.022245 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:44.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:44.022588 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:45.025830 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:45.026124 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:45.207 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:45.208 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:45.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:45.210 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:45.210 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:45.210 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:45.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:45.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:45.212 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:45.988507 26180 master_runner.cc:186] Error getting master registration for 127.25.55.126:41357: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111),
01:49:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:45.988916 26180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:46.029372 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:46.029654 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:47.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:47.032454 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:47.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:47.032730 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:48.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:48.035537 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:48.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:48.035804 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:49.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:49.038683 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:49.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:49.038954 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:49.227 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:49.229 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:49.230 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:49.230 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:49.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:49.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:49.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:49.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:49.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:49.547 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:49.548 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:49.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:49.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:49.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:49.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:49.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:49.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:49.552 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:49.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:49.647961 26402 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:49:49.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:49.648579 26402 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8f3c5485ccb4490a2ea7dad5870c3af" member_type: VOTER last_known_addr { host: "127.25.55.126" port: 41357 } } peers { permanent_uuid: "1949606339324dc58c371835ff561fdc" member_type: VOTER last_known_addr { host: "127.25.55.125" port: 37313 } } peers { permanent_uuid: "7464af44d83e4841b6113e1c7032c358" member_type: VOTER last_known_addr { host: "127.25.55.124" port: 42341 } }
01:49:49.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:49.650391 26402 leader_election.cc:288] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357), 1949606339324dc58c371835ff561fdc (127.25.55.125:37313)
01:49:49.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:49.652930 26204 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:49:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:49.656062 26202 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1949606339324dc58c371835ff561fdc (127.25.55.125:37313): Network error: Client connection negotiation failed: client connection to 127.25.55.125:37313: connect: Connection refused (error 111)
01:49:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:49.656527 26204 leader_election.cc:334] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d8f3c5485ccb4490a2ea7dad5870c3af (127.25.55.126:41357): Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111)
01:49:49.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:49.656848 26204 leader_election.cc:302] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7464af44d83e4841b6113e1c7032c358; no voters: 1949606339324dc58c371835ff561fdc, d8f3c5485ccb4490a2ea7dad5870c3af
01:49:49.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:49.657529 26402 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:49:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:50.042495 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:50.042834 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:49:50 dist-test-slave-s8kj krb5kdc[25833](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006590, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.55.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:49:50.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:50.635128 26196 init.cc:295] Successfully reacquired a new kerberos TGT
01:49:50.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:50.991779 26180 master_runner.cc:186] Error getting master registration for 127.25.55.126:41357: Network error: Client connection negotiation failed: client connection to 127.25.55.126:41357: connect: Connection refused (error 111),
01:49:50.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:50.992179 26180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:49:51.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:51.045594 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:51.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:51.045897 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:52.048448 26267 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7464af44d83e4841b6113e1c7032c358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:49:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:49:52.048741 26267 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:49:52.207 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.126:41357(127.25.55.126:41357)] connecting to peer
01:49:52.208 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.55.125:37313(127.25.55.125:37313)] connecting to peer
01:49:52.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:52.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:52.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.126:41357(127.25.55.126:41357)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:52.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.55.125:37313(127.25.55.125:37313)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:52.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.126:41357: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357
01:49:52.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.55.125:37313: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
01:49:52.212 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313; will retry
01:49:52.217 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29439), Traces: [0ms] refreshing cache from master, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [227ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [229ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [541ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [554ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [563ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [564ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [565ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [580ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [582ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [584ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [604ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [620ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [621ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [622ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [662ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [734ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [743ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [879ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [880ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [881ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [882ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1022ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1254ms] refreshing cache from master, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1260ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1261ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2115ms] refreshing cache from master, [2115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [2116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [2117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2121ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [2121ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4935ms] refreshing cache from master, [4935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [4936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [4939ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4940ms] too many traces: truncated at 100 traces, deferred=Deferred@125866059(state=PENDING, result=null, callback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440), errback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440)))
01:49:52.222 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.55.126:41357(127.25.55.126:41357)] disconnecting while in state TERMINATED
01:49:52.223 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.55.125:37313(127.25.55.125:37313)] disconnecting while in state TERMINATED
01:49:52.223 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.55.124:42341(127.25.55.124:42341)] disconnecting while in state READY
01:49:52.229 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:49:52.231 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.25.55.124:42341(127.25.55.124:42341)] handling channelInactive
01:49:52.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.55.124:42341(127.25.55.124:42341)] cleaning up while in state READY due to: connection closed
01:49:52.234 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:49:52.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:52.237924 25821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 26180
01:49:53.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:49:53Z chronyd exiting
01:49:54.468 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:49:54.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:54.469615 25094 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 25375
01:49:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:54.496608 25094 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 25533
01:49:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:54.524900 25094 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 25680
01:49:54.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:54.551466 25094 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 25106
01:49:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:54.582756 25094 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 25176
01:49:54.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:49:54.617905 25094 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHBGiCK/build/tsan/bin/kudu with pid 25251
01:49:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:49:55Z chronyd exiting
01:49:56.851 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29439), Traces: [0ms] refreshing cache from master, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [227ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [229ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [541ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [554ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [563ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [564ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [565ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [580ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [582ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [584ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [604ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [620ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [621ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [622ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [662ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [734ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [743ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [879ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [880ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [881ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [882ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1022ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1254ms] refreshing cache from master, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1260ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1261ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2115ms] refreshing cache from master, [2115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [2116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [2117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2121ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [2121ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4935ms] refreshing cache from master, [4935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [4936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [4939ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4940ms] too many traces: truncated at 100 traces, deferred=Deferred@125866059(state=PENDING, result=null, callback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440), errback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.55.125:37313
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29439), Traces: [0ms] refreshing cache from master, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [227ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [229ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [541ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [554ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [563ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [564ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [565ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [580ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [582ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [584ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [604ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [620ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [621ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [622ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [662ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [734ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [743ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [879ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [880ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [881ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [882ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1022ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1254ms] refreshing cache from master, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1260ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1261ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2115ms] refreshing cache from master, [2115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [2116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [2117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2121ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [2121ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4935ms] refreshing cache from master, [4935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [4936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [4939ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4940ms] too many traces: truncated at 100 traces, deferred=Deferred@125866059(state=PENDING, result=null, callback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440), errback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.55.125:37313
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
01:49:57.450 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
01:49:57.451 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29439), Traces: [0ms] refreshing cache from master, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [227ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [229ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [541ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [554ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [563ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [564ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [565ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [580ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [582ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [584ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [604ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [620ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [621ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [622ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [662ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [734ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [743ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [879ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [880ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [881ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [882ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1022ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1254ms] refreshing cache from master, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1260ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1261ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2115ms] refreshing cache from master, [2115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [2116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [2117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2121ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [2121ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4935ms] refreshing cache from master, [4935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [4936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [4939ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4940ms] too many traces: truncated at 100 traces, deferred=Deferred@125866059(state=PENDING, result=null, callback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440), errback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.55.125:37313
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29439), Traces: [0ms] refreshing cache from master, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [185ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [227ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [229ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [541ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [554ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [563ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [564ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [565ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [580ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [582ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [584ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [600ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [604ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [620ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [621ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [622ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [660ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [662ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [734ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [743ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [819ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [822ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [879ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [880ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [881ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [882ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1022ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1254ms] refreshing cache from master, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [1260ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [1261ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2115ms] refreshing cache from master, [2115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [2116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [2117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.125:37313, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [2119ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.125:37313: Network error: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313, [2121ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.124:42341: OK, [2121ms] delaying RPC due to: Service unavailable: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4935ms] refreshing cache from master, [4935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.126:41357, [4936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.55.124:42341, [4939ms] Sub RPC ConnectToMaster: received response from server master-127.25.55.126:41357: Network error: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357, [4940ms] too many traces: truncated at 100 traces, deferred=Deferred@125866059(state=PENDING, result=null, callback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440), errback=(continuation of Deferred@640455114 after retry RPC after error@1142381559) -> (continuation of Deferred@1028620988 after retry RPC after error@1045323670) -> (continuation of Deferred@2091789140 after retry RPC after error@43524053) -> (continuation of Deferred@1124280152 after retry RPC after error@259875393) -> (continuation of Deferred@33823857 after retry RPC after error@1084284243) -> (continuation of Deferred@440601592 after retry RPC after error@2127043995) -> (continuation of Deferred@1552769447 after retry RPC after error@444691744) -> (continuation of Deferred@404730179 after retry RPC after error@637883166) -> (continuation of Deferred@288525856 after retry RPC after error@9158995) -> (continuation of Deferred@1711013288 after retry RPC after error@1768611754) -> (continuation of Deferred@1814826793 after retry RPC after error@854837926) -> (continuation of Deferred@148862957 after retry RPC after error@938966739) -> (continuation of Deferred@992897603 after retry RPC after error@396632442) -> (continuation of Deferred@1689734268 after retry RPC after error@1748678623) -> (continuation of Deferred@546223337 after retry RPC after error@379059827) -> (continuation of Deferred@1209894001 after retry RPC after error@1965311714) -> (continuation of Deferred@551062867 after retry RPC after error@152406877) -> (continuation of Deferred@764389378 after retry RPC after error@2051794951) -> (continuation of Deferred@505237512 after retry RPC after error@997687285) -> (continuation of Deferred@1810370337 after retry RPC after error@1313266059) -> (continuation of Deferred@976178523 after retry RPC after error@975639868) -> (continuation of Deferred@295539806 after retry RPC after error@1777306983) -> (continuation of Deferred@703221908 after retry RPC after error@736505440)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.55.126:41357,127.25.55.124:42341,127.25.55.125:37313) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.126:41357(127.25.55.126:41357): Connection refused: /127.25.55.126:41357,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.55.125:37313(127.25.55.125:37313): Connection refused: /127.25.55.125:37313
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.55.125:37313
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)