20:36:09.992 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:36:10.005 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskm_a55n/test-tmp
20:36:10.007 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:36:10.011 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskm_a55n/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:36:10.309 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:36:10.459 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:36:10.460 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:36:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.464058 15046 mini_oidc.cc:152] Starting JWKS server
20:36:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.468375 15046 webserver.cc:460] Webserver started at http://127.0.0.1:39347/ using document root <none> and password file <none>
20:36:10.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.468869 15046 mini_oidc.cc:189] Starting OIDC Discovery server
20:36:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.470755 15046 webserver.cc:460] Webserver started at http://127.0.0.1:42683/ using document root <none> and password file <none>
20:36:10.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:36:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:36:10.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:36:10Z Disabled control of system clock
20:36:10.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.522385 15046 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/wal
20:36:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/data
20:36:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/logs
20:36:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/data/info.pb
20:36:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.190:36727
20:36:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.190
20:36:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39347/jwks.json with env {}
20:36:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.824678 15058 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.825242 15058 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.825909 15058 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.857028 15058 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.857319 15058 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.857578 15058 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.857806 15058 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.858088 15058 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.858320 15058 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39347/jwks.json
20:36:10.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.893224 15058 master_runner.cc:386] Master server non-default flags:
20:36:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/data
20:36:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/wal
20:36:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.190:36727
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39347/jwks.json
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/data/info.pb
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.190
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/logs
20:36:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.894606 15058 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.896737 15058 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:10.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.914973 15065 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:12.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.915159 15064 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:12.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:10.915654 15067 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:12.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:10.917021 15058 server_base.cc:1034] running on GCE node
20:36:12.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.107003 15058 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.110284 15058 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.111724 15058 hybrid_clock.cc:648] HybridClock initialized: now 1786048572111677 us; error 94 us; skew 500 ppm
20:36:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.112550 15058 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:12.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.119235 15058 webserver.cc:460] Webserver started at http://127.14.177.190:38487/ using document root <none> and password file <none>
20:36:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.120123 15058 fs_manager.cc:359] Metadata directory not provided
20:36:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.120302 15058 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.120680 15058 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.125439 15058 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/data/instance:
20:36:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f5f6895ab564505b1ad9facf5c57977"
20:36:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:12 on dist-test-slave-bt4h"
20:36:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.126654 15058 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/wal/instance:
20:36:12.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f5f6895ab564505b1ad9facf5c57977"
20:36:12.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:12 on dist-test-slave-bt4h"
20:36:12.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.133826 15058 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
20:36:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.139482 15074 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.140586 15058 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:36:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.140965 15058 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/wal
20:36:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f5f6895ab564505b1ad9facf5c57977"
20:36:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:12 on dist-test-slave-bt4h"
20:36:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.141376 15058 fs_report.cc:383] FS layout report
20:36:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/wal
20:36:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/wal
20:36:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/data/data
20:36:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.179783 15058 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.181449 15058 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.181994 15058 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.255036 15058 rpc_server.cc:307] RPC server started. Bound to: 127.14.177.190:36727
20:36:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.255103 15128 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.177.190:36727 every 8 connection(s)
20:36:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.258067 15058 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/data/info.pb
20:36:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.263604 15129 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:36:12.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.266039 15046 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15058
20:36:12.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.266429 15046 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-0/wal/instance
20:36:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.270780 15046 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/wal
20:36:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/data
20:36:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/logs
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/data/info.pb
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:12.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:12.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.189:42485
20:36:12.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.189
20:36:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:12.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39347/jwks.json with env {}
20:36:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.277210 15129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } has no permanent_uuid. Determining permanent_uuid...
20:36:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.308141 15129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } has no permanent_uuid. Determining permanent_uuid...
20:36:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.310842 15075 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.177.189:42485: connect: Connection refused (error 111)
20:36:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.315317 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.189:42485: Network error: Client connection negotiation failed: client connection to 127.14.177.189:42485: connect: Connection refused (error 111)
20:36:12.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.364869 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } attempt: 1
20:36:12.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.368770 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.189:42485: Network error: Client connection negotiation failed: client connection to 127.14.177.189:42485: connect: Connection refused (error 111)
20:36:12.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.437286 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } attempt: 2
20:36:12.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.441121 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.189:42485: Network error: Client connection negotiation failed: client connection to 127.14.177.189:42485: connect: Connection refused (error 111)
20:36:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.532588 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } attempt: 3
20:36:12.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.536604 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.189:42485: Network error: Client connection negotiation failed: client connection to 127.14.177.189:42485: connect: Connection refused (error 111)
20:36:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.583001 15131 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.583732 15131 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.584408 15131 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:12.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.615453 15131 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.615831 15131 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.616164 15131 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.616521 15131 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.616849 15131 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.617127 15131 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39347/jwks.json
20:36:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.651962 15131 master_runner.cc:386] Master server non-default flags:
20:36:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:12.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/data
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/wal
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:12.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.189:42485
20:36:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39347/jwks.json
20:36:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/data/info.pb
20:36:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.189
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/logs
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.653606 15131 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.655912 15131 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:12.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.675151 15140 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:12.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.679975 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } attempt: 4
20:36:12.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.683508 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.189:42485: Network error: Client connection negotiation failed: client connection to 127.14.177.189:42485: connect: Connection refused (error 111)
20:36:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.982986 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } attempt: 5
20:36:12.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.987059 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.189:42485: Network error: Client connection negotiation failed: client connection to 127.14.177.189:42485: connect: Connection refused (error 111)
20:36:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.534780 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } attempt: 6
20:36:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:13.543551 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.189:42485: Network error: Client connection negotiation failed: client connection to 127.14.177.189:42485: connect: Connection refused (error 111)
20:36:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.675434 15143 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:12.676920 15131 server_base.cc:1034] running on GCE node
20:36:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:12.676301 15141 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:13.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.963179 15131 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:13.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.966246 15131 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.967705 15131 hybrid_clock.cc:648] HybridClock initialized: now 1786048573967638 us; error 110 us; skew 500 ppm
20:36:13.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.969050 15131 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.975869 15131 webserver.cc:460] Webserver started at http://127.14.177.189:42185/ using document root <none> and password file <none>
20:36:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.976907 15131 fs_manager.cc:359] Metadata directory not provided
20:36:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.977093 15131 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.977483 15131 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.982674 15131 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/data/instance:
20:36:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa114dc61def4dcda00f853b0c459af7"
20:36:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:13 on dist-test-slave-bt4h"
20:36:13.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.984090 15131 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/wal/instance:
20:36:13.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa114dc61def4dcda00f853b0c459af7"
20:36:13.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:13 on dist-test-slave-bt4h"
20:36:13.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.992066 15131 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
20:36:13.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.997895 15153 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.999091 15131 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
20:36:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.999464 15131 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/wal
20:36:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa114dc61def4dcda00f853b0c459af7"
20:36:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:13 on dist-test-slave-bt4h"
20:36:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:13.999846 15131 fs_report.cc:383] FS layout report
20:36:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/wal
20:36:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/wal
20:36:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/data/data
20:36:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:14.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.042333 15131 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.044028 15131 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.044587 15131 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:14.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.123862 15131 rpc_server.cc:307] RPC server started. Bound to: 127.14.177.189:42485
20:36:14.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.123974 15205 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.177.189:42485 every 8 connection(s)
20:36:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.126817 15131 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/data/info.pb
20:36:14.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.129951 15046 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15131
20:36:14.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.130434 15046 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-1/wal/instance
20:36:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.133236 15206 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:36:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.133528 15046 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/wal
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/data
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/logs
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/data/info.pb
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.188:45045
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.188
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39347/jwks.json with env {}
20:36:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.146793 15206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } has no permanent_uuid. Determining permanent_uuid...
20:36:14.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.170979 15206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } has no permanent_uuid. Determining permanent_uuid...
20:36:14.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.187816 15206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } has no permanent_uuid. Determining permanent_uuid...
20:36:14.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.190310 15154 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.198400 15206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.248018 15206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 1
20:36:14.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.252640 15206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.321113 15206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 2
20:36:14.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.325320 15206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.416749 15206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 3
20:36:14.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.420575 15206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.457445 15208 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:14.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.458178 15208 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:14.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.458909 15208 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.490998 15208 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.491394 15208 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.491719 15208 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.492067 15208 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.492399 15208 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.492686 15208 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39347/jwks.json
20:36:14.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.533171 15208 master_runner.cc:386] Master server non-default flags:
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/data
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/wal
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.188:45045
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39347/jwks.json
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/data/info.pb
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.188
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/logs
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.534786 15208 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:14.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.537025 15208 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.555640 15219 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:14.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.564062 15206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 4
20:36:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.567871 15206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.604013 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } attempt: 7
20:36:14.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.623148 15129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } has no permanent_uuid. Determining permanent_uuid...
20:36:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.630115 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.688558 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 1
20:36:14.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.693496 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.774922 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 2
20:36:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.779621 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.865023 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 3
20:36:14.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.867331 15206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 5
20:36:14.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.871428 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:14.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.871948 15206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.011951 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 4
20:36:15.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:15.017040 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.300467 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 5
20:36:15.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:15.304793 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.419431 15206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 6
20:36:15.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:15.427543 15206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.555644 15218 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:14.555732 15221 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:14.557600 15208 server_base.cc:1034] running on GCE node
20:36:15.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.814426 15208 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.817117 15208 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.818568 15208 hybrid_clock.cc:648] HybridClock initialized: now 1786048575818490 us; error 39 us; skew 500 ppm
20:36:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.819530 15208 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:15.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.826090 15208 webserver.cc:460] Webserver started at http://127.14.177.188:45641/ using document root <none> and password file <none>
20:36:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.827198 15208 fs_manager.cc:359] Metadata directory not provided
20:36:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.827452 15208 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:15.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.828029 15208 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.833276 15208 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/data/instance:
20:36:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3"
20:36:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:15 on dist-test-slave-bt4h"
20:36:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.834610 15208 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/wal/instance:
20:36:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3"
20:36:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:15 on dist-test-slave-bt4h"
20:36:15.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.842811 15208 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:36:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.849331 15232 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.850886 15208 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.004s
20:36:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.851315 15208 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/wal
20:36:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3"
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:15 on dist-test-slave-bt4h"
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.851825 15208 fs_report.cc:383] FS layout report
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/wal
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/wal
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/data/data
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:15.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:15.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:15.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.857439 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 6
20:36:15.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:15.864262 15129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.177.188:45045: Network error: Client connection negotiation failed: client connection to 127.14.177.188:45045: connect: Connection refused (error 111)
20:36:15.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.891196 15208 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.892712 15208 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.893148 15208 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.967000 15208 rpc_server.cc:307] RPC server started. Bound to: 127.14.177.188:45045
20:36:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.967113 15288 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.177.188:45045 every 8 connection(s)
20:36:15.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.970798 15208 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/data/info.pb
20:36:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.972666 15046 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15208
20:36:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.973227 15046 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/master-2/wal/instance
20:36:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.977921 15289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:36:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:15.993300 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } has no permanent_uuid. Determining permanent_uuid...
20:36:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.023129 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } has no permanent_uuid. Determining permanent_uuid...
20:36:16.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.034466 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } has no permanent_uuid. Determining permanent_uuid...
20:36:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.053155 15289 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3: Bootstrap starting.
20:36:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.059165 15289 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3: Neither blocks nor log segments found. Creating new log.
20:36:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.061834 15289 log.cc:824] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3: Log is configured to *not* fsync() on all Append() calls
20:36:16.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.068933 15289 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3: No bootstrap required, opened a new log
20:36:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.103180 15289 raft_consensus.cc:348] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:16.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.103806 15289 raft_consensus.cc:374] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:36:16.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.104007 15289 raft_consensus.cc:729] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f4fbe4c60ccc40fa9ab5b0d0cf9770a3, State: Initialized, Role: FOLLOWER
20:36:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.104635 15289 consensus_queue.cc:260] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.107124 15298 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.107883 15298 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: This master's current role is: FOLLOWER
20:36:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.108373 15289 sys_catalog.cc:564] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: configured and running, proceeding with master startup.
20:36:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:16.142073 15309 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:16.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:16.143692 15309 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:16.148316 15240 tablet.cc:2369] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:36:16.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.134819 15208 master_runner.cc:186] Error getting master registration for 127.14.177.190:36727: OK, instance_id { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" instance_seqno: 1786048572234128 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:36:16.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.166064 15208 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:36:16.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.342271 15298 raft_consensus.cc:482] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:36:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.342898 15298 raft_consensus.cc:504] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.346443 15298 leader_election.cc:288] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7f5f6895ab564505b1ad9facf5c57977 (127.14.177.190:36727), fa114dc61def4dcda00f853b0c459af7 (127.14.177.189:42485)
20:36:16.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.487977 15206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 7
20:36:16.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.517689 15206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: Bootstrap starting.
20:36:16.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.527451 15206 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: Neither blocks nor log segments found. Creating new log.
20:36:16.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.530840 15206 log.cc:824] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: Log is configured to *not* fsync() on all Append() calls
20:36:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.539815 15206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: No bootstrap required, opened a new log
20:36:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.579149 15206 raft_consensus.cc:348] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:16.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.580595 15206 raft_consensus.cc:374] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:36:16.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.580911 15206 raft_consensus.cc:729] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa114dc61def4dcda00f853b0c459af7, State: Initialized, Role: FOLLOWER
20:36:16.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.582319 15206 consensus_queue.cc:260] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:16.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.589123 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:16.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.591571 15206 sys_catalog.cc:564] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [sys.catalog]: configured and running, proceeding with master startup.
20:36:16.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.591742 15181 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa114dc61def4dcda00f853b0c459af7" is_pre_election: true
20:36:16.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.592633 15181 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f4fbe4c60ccc40fa9ab5b0d0cf9770a3 in term 0.
20:36:16.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.595001 15235 leader_election.cc:302] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: f4fbe4c60ccc40fa9ab5b0d0cf9770a3, fa114dc61def4dcda00f853b0c459af7; no voters:
20:36:16.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.596498 15298 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:36:16.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.596939 15298 raft_consensus.cc:482] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:36:16.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.597353 15298 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 0 FOLLOWER]: Advancing to term 1
20:36:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.591424 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [sys.catalog]: This master's current role is: FOLLOWER
20:36:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.612612 15298 raft_consensus.cc:504] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:16.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.620078 15181 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa114dc61def4dcda00f853b0c459af7"
20:36:16.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.620929 15181 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [term 0 FOLLOWER]: Advancing to term 1
20:36:16.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.630746 15298 leader_election.cc:288] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [CANDIDATE]: Term 1 election: Requested vote from peers 7f5f6895ab564505b1ad9facf5c57977 (127.14.177.190:36727), fa114dc61def4dcda00f853b0c459af7 (127.14.177.189:42485)
20:36:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.634254 15181 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f4fbe4c60ccc40fa9ab5b0d0cf9770a3 in term 1.
20:36:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.636900 15235 leader_election.cc:302] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: f4fbe4c60ccc40fa9ab5b0d0cf9770a3, fa114dc61def4dcda00f853b0c459af7; no voters:
20:36:16.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.638590 15298 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 1 FOLLOWER]: Leader election won for term 1
20:36:16.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.649369 15298 raft_consensus.cc:686] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [term 1 LEADER]: Becoming Leader. State: Replica: f4fbe4c60ccc40fa9ab5b0d0cf9770a3, State: Running, Role: LEADER
20:36:16.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.653000 15298 consensus_queue.cc:237] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:16.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.676505 15318 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4fbe4c60ccc40fa9ab5b0d0cf9770a3. Latest consensus state: current_term: 1 leader_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:16.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.678207 15318 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: This master's current role is: LEADER
20:36:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.683781 15324 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:36:16.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.695470 15324 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:36:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.738577 15181 raft_consensus.cc:1261] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [term 1 FOLLOWER]: Refusing update from remote peer f4fbe4c60ccc40fa9ab5b0d0cf9770a3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:36:16.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:16.738497 15327 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:16.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.741164 15318 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:36:16.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:16.742233 15327 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:16.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.708937 15131 master_runner.cc:186] Error getting master registration for 127.14.177.190:36727: OK, instance_id { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" instance_seqno: 1786048572234128 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:36:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.790681 15131 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:36:16.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:16.803393 15159 tablet.cc:2369] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:36:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.814271 15328 mvcc.cc:205] Tried to move back new op lower bound from 7315654970282037248 to 7315654970019188736. Current Snapshot: MvccSnapshot[applied={T|T < 7315654970282037248}]
20:36:16.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.824543 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4fbe4c60ccc40fa9ab5b0d0cf9770a3. Latest consensus state: current_term: 1 leader_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:16.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.826035 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [sys.catalog]: This master's current role is: FOLLOWER
20:36:16.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.827175 15318 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:16.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.828465 15318 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: This master's current role is: LEADER
20:36:16.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.833230 15324 catalog_manager.cc:1347] Generated new cluster ID: 6f18d4a598674ddcb8668b29a61488f9
20:36:16.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.833523 15324 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:36:16.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.860489 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.861756 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7 [sys.catalog]: This master's current role is: FOLLOWER
20:36:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.890901 15324 catalog_manager.cc:1370] Generated new certificate authority record
20:36:16.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.897665 15129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } attempt: 7
20:36:16.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.912909 15324 catalog_manager.cc:1504] Loading token signing keys...
20:36:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.988132 15129 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977: Bootstrap starting.
20:36:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:16.992899 15324 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3: Generated new TSK 0
20:36:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.000648 15324 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:36:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.033037 15129 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977: Neither blocks nor log segments found. Creating new log.
20:36:17.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.036098 15129 log.cc:824] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977: Log is configured to *not* fsync() on all Append() calls
20:36:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.057284 15046 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/wal
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/data
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/logs
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/data/info.pb
20:36:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.129:0
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.177.129
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.129
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:17.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:36:17.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.079372 15129 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977: No bootstrap required, opened a new log
20:36:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.148662 15129 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.149588 15129 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:36:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.150018 15129 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f5f6895ab564505b1ad9facf5c57977, State: Initialized, Role: FOLLOWER
20:36:17.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.159055 15129 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } }
20:36:17.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.173329 15338 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:17.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.174279 15338 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [sys.catalog]: This master's current role is: FOLLOWER
20:36:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.179183 15129 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [sys.catalog]: configured and running, proceeding with master startup.
20:36:17.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.179827 15101 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f5f6895ab564505b1ad9facf5c57977"
20:36:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.180699 15101 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [term 0 FOLLOWER]: Advancing to term 1
20:36:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.183277 15102 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f5f6895ab564505b1ad9facf5c57977" is_pre_election: true
20:36:17.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:17.193354 15080 tablet.cc:2369] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:36:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.208863 15101 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f4fbe4c60ccc40fa9ab5b0d0cf9770a3 in term 1.
20:36:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.211215 15099 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [term 1 FOLLOWER]: Refusing update from remote peer f4fbe4c60ccc40fa9ab5b0d0cf9770a3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:36:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.223224 15318 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:36:17.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.388032 15338 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4fbe4c60ccc40fa9ab5b0d0cf9770a3. Latest consensus state: current_term: 1 leader_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.388808 15338 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [sys.catalog]: This master's current role is: FOLLOWER
20:36:17.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.390129 15348 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:17.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.390990 15348 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977 [sys.catalog]: This master's current role is: FOLLOWER
20:36:17.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.467509 15355 catalog_manager.cc:1259] Loaded cluster ID: 6f18d4a598674ddcb8668b29a61488f9
20:36:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.467798 15355 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977: loading cluster ID for follower catalog manager: success
20:36:17.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.499678 15355 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977: acquiring CA information for follower catalog manager: success
20:36:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.517432 15355 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 7f5f6895ab564505b1ad9facf5c57977: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:36:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.754796 15327 catalog_manager.cc:1259] Loaded cluster ID: 6f18d4a598674ddcb8668b29a61488f9
20:36:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.755539 15327 catalog_manager.cc:1552] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: loading cluster ID for follower catalog manager: success
20:36:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.775424 15327 catalog_manager.cc:1574] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: acquiring CA information for follower catalog manager: success
20:36:17.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.783164 15327 catalog_manager.cc:1602] T 00000000000000000000000000000000 P fa114dc61def4dcda00f853b0c459af7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:36:17.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.810011 15318 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f5f6895ab564505b1ad9facf5c57977" member_type: VOTER last_known_addr { host: "127.14.177.190" port: 36727 } } peers { permanent_uuid: "fa114dc61def4dcda00f853b0c459af7" member_type: VOTER last_known_addr { host: "127.14.177.189" port: 42485 } } peers { permanent_uuid: "f4fbe4c60ccc40fa9ab5b0d0cf9770a3" member_type: VOTER last_known_addr { host: "127.14.177.188" port: 45045 } } }
20:36:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:17.811051 15318 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4fbe4c60ccc40fa9ab5b0d0cf9770a3 [sys.catalog]: This master's current role is: LEADER
20:36:17.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:17.924322 15337 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:17.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:17.925374 15337 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:17.926575 15337 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:17.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:17.998191 15337 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:18.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:18.000133 15337 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.177.129
20:36:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:18.071099 15337 tablet_server_runner.cc:78] Tablet server non-default flags:
20:36:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/data
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/wal
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.129:0
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/data/info.pb
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.129
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.177.129
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/logs
20:36:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:18.074981 15337 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:18.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:18.082640 15337 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:18.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:18.142150 15366 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:19.512655 15364 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 15337
20:36:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:20.577482 15364 kernel_stack_watchdog.cc:198] Thread 15337 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 378ms:
20:36:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:36:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:36:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:36:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:36:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:20.577756 15337 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.446s user 0.588s sys 0.898s
20:36:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:18.146970 15365 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:20.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:20.578706 15337 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.447s user 0.589s sys 0.898s
20:36:20.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.591657 15337 server_base.cc:1034] running on GCE node
20:36:20.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:20.591647 15373 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:20.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.593415 15337 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:20.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.596016 15337 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:20.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.597481 15337 hybrid_clock.cc:648] HybridClock initialized: now 1786048580597326 us; error 154 us; skew 500 ppm
20:36:20.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.598572 15337 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.609510 15337 webserver.cc:460] Webserver started at http://127.14.177.129:39591/ using document root <none> and password file <none>
20:36:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.610752 15337 fs_manager.cc:359] Metadata directory not provided
20:36:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.611024 15337 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.611557 15337 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.619534 15337 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/data/instance:
20:36:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e56d4bdbe8741e6b176c7d0b585287a"
20:36:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:20 on dist-test-slave-bt4h"
20:36:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.621028 15337 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/wal/instance:
20:36:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e56d4bdbe8741e6b176c7d0b585287a"
20:36:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:20 on dist-test-slave-bt4h"
20:36:20.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.633193 15337 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.013s sys 0.000s
20:36:20.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.643348 15378 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:20.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.644809 15337 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.007s sys 0.001s
20:36:20.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.645205 15337 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/wal
20:36:20.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e56d4bdbe8741e6b176c7d0b585287a"
20:36:20.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:20 on dist-test-slave-bt4h"
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.645632 15337 fs_report.cc:383] FS layout report
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/wal
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/wal
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/data/data
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.721334 15337 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:20.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.724066 15337 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:20.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.725188 15337 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:20.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.731230 15337 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:36:20.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.739202 15337 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:36:20.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.739532 15337 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:36:20.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.739833 15337 ts_tablet_manager.cc:610] Registered 0 tablets
20:36:20.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:20.740024 15337 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:36:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.033346 15337 rpc_server.cc:307] RPC server started. Bound to: 127.14.177.129:37107
20:36:21.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.033463 15495 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.177.129:37107 every 8 connection(s)
20:36:21.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.036285 15337 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/data/info.pb
20:36:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.046506 15046 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15337
20:36:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.047888 15046 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-0/wal/instance
20:36:21.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.067106 15046 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:21.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/wal
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/data
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/logs
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/data/info.pb
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.130:0
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.177.130
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.130
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:36:21.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.112677 15496 heartbeater.cc:347] Connected to a master server at 127.14.177.188:45045
20:36:21.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.113224 15496 heartbeater.cc:464] Registering TS with master...
20:36:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.113920 15499 heartbeater.cc:347] Connected to a master server at 127.14.177.190:36727
20:36:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.114324 15499 heartbeater.cc:464] Registering TS with master...
20:36:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.114924 15496 heartbeater.cc:511] Master 127.14.177.188:45045 requested a full tablet report, sending...
20:36:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.115381 15499 heartbeater.cc:511] Master 127.14.177.190:36727 requested a full tablet report, sending...
20:36:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.119253 15091 ts_manager.cc:194] Registered new tserver with Master: 6e56d4bdbe8741e6b176c7d0b585287a (127.14.177.129:37107)
20:36:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.119475 15252 ts_manager.cc:194] Registered new tserver with Master: 6e56d4bdbe8741e6b176c7d0b585287a (127.14.177.129:37107)
20:36:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.123288 15252 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.177.129:36757
20:36:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.131639 15497 heartbeater.cc:347] Connected to a master server at 127.14.177.189:42485
20:36:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.132138 15497 heartbeater.cc:464] Registering TS with master...
20:36:21.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.133226 15497 heartbeater.cc:511] Master 127.14.177.189:42485 requested a full tablet report, sending...
20:36:21.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.136701 15170 ts_manager.cc:194] Registered new tserver with Master: 6e56d4bdbe8741e6b176c7d0b585287a (127.14.177.129:37107)
20:36:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:21.517570 15506 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:21.518117 15506 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:21.518671 15506 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:21.549687 15506 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:21.550490 15506 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.177.130
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.586856 15506 tablet_server_runner.cc:78] Tablet server non-default flags:
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/data
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/wal
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:21.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.130:0
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/data/info.pb
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.130
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.177.130
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/logs
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.588446 15506 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.590513 15506 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:21.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:21.617499 15513 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:22.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.128324 15496 heartbeater.cc:503] Master 127.14.177.188:45045 was elected leader, sending a full tablet report...
20:36:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:21.617868 15516 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:22.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:21.618183 15514 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:22.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:21.619724 15506 server_base.cc:1034] running on GCE node
20:36:22.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.845224 15506 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.847570 15506 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:22.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.848924 15506 hybrid_clock.cc:648] HybridClock initialized: now 1786048582848894 us; error 44 us; skew 500 ppm
20:36:22.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.849664 15506 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.856716 15506 webserver.cc:460] Webserver started at http://127.14.177.130:33695/ using document root <none> and password file <none>
20:36:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.857617 15506 fs_manager.cc:359] Metadata directory not provided
20:36:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.857818 15506 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.858278 15506 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.863340 15506 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/data/instance:
20:36:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47be493fcd104a57a4390300e96168cf"
20:36:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:22 on dist-test-slave-bt4h"
20:36:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.864423 15506 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/wal/instance:
20:36:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47be493fcd104a57a4390300e96168cf"
20:36:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:22 on dist-test-slave-bt4h"
20:36:22.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.871613 15506 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
20:36:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.877339 15524 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:22.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.878362 15506 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
20:36:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.878764 15506 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/wal
20:36:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47be493fcd104a57a4390300e96168cf"
20:36:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:22 on dist-test-slave-bt4h"
20:36:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.879117 15506 fs_report.cc:383] FS layout report
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/wal
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/wal
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/data/data
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:22.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.917168 15506 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:22.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.918756 15506 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:22.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.919363 15506 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:22.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.922009 15506 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:36:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.926132 15506 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:36:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.926396 15506 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:36:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.926771 15506 ts_tablet_manager.cc:610] Registered 0 tablets
20:36:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:22.926985 15506 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:36:23.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.225353 15506 rpc_server.cc:307] RPC server started. Bound to: 127.14.177.130:42175
20:36:23.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.225449 15638 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.177.130:42175 every 8 connection(s)
20:36:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.228436 15506 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/data/info.pb
20:36:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.234050 15046 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15506
20:36:23.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.234480 15046 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-1/wal/instance
20:36:23.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.247709 15046 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:23.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/wal
20:36:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/data
20:36:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/logs
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/data/info.pb
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.131:0
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.177.131
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.131
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:36:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.315780 15639 heartbeater.cc:347] Connected to a master server at 127.14.177.188:45045
20:36:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.316337 15639 heartbeater.cc:464] Registering TS with master...
20:36:23.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.317638 15639 heartbeater.cc:511] Master 127.14.177.188:45045 requested a full tablet report, sending...
20:36:23.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.318109 15643 heartbeater.cc:347] Connected to a master server at 127.14.177.190:36727
20:36:23.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.318542 15643 heartbeater.cc:464] Registering TS with master...
20:36:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.319576 15643 heartbeater.cc:511] Master 127.14.177.190:36727 requested a full tablet report, sending...
20:36:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.321776 15252 ts_manager.cc:194] Registered new tserver with Master: 47be493fcd104a57a4390300e96168cf (127.14.177.130:42175)
20:36:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.322582 15091 ts_manager.cc:194] Registered new tserver with Master: 47be493fcd104a57a4390300e96168cf (127.14.177.130:42175)
20:36:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.323838 15252 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.177.130:57109
20:36:23.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.327342 15640 heartbeater.cc:347] Connected to a master server at 127.14.177.189:42485
20:36:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.327855 15640 heartbeater.cc:464] Registering TS with master...
20:36:23.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.331492 15640 heartbeater.cc:511] Master 127.14.177.189:42485 requested a full tablet report, sending...
20:36:23.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.335884 15170 ts_manager.cc:194] Registered new tserver with Master: 47be493fcd104a57a4390300e96168cf (127.14.177.130:42175)
20:36:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:23.705735 15649 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:23.706244 15649 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:23.706707 15649 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:23.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:23.738411 15649 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:23.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:23.739255 15649 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.177.131
20:36:23.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.775451 15649 tablet_server_runner.cc:78] Tablet server non-default flags:
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.177.148:40507
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/data
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/wal
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.177.131:0
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/data/info.pb
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.177.131
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.177.190:36727,127.14.177.189:42485,127.14.177.188:45045
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.177.131
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/logs
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.776731 15649 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:23.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.778338 15649 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:23.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:23.807286 15659 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:24.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:24.335063 15639 heartbeater.cc:503] Master 127.14.177.188:45045 was elected leader, sending a full tablet report...
20:36:25.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:23.808867 15657 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:25.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:23.814318 15649 server_base.cc:1034] running on GCE node
20:36:25.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:23.810510 15656 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.039844 15649 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:25.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.042826 15649 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:25.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.044341 15649 hybrid_clock.cc:648] HybridClock initialized: now 1786048585044281 us; error 73 us; skew 500 ppm
20:36:25.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.045157 15649 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.055902 15649 webserver.cc:460] Webserver started at http://127.14.177.131:38027/ using document root <none> and password file <none>
20:36:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.056813 15649 fs_manager.cc:359] Metadata directory not provided
20:36:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.057004 15649 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.057404 15649 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:25.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.062877 15649 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/data/instance:
20:36:25.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe9fa4621124370a49453e468714be5"
20:36:25.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:25 on dist-test-slave-bt4h"
20:36:25.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.064314 15649 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/wal/instance:
20:36:25.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe9fa4621124370a49453e468714be5"
20:36:25.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:25 on dist-test-slave-bt4h"
20:36:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.071982 15649 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:36:25.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.081314 15667 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:25.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.082515 15649 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.002s
20:36:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.082896 15649 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/wal
20:36:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe9fa4621124370a49453e468714be5"
20:36:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:25 on dist-test-slave-bt4h"
20:36:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.083251 15649 fs_report.cc:383] FS layout report
20:36:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/wal
20:36:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/wal
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/data/data
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:25.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.121884 15649 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.123855 15649 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.124444 15649 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:25.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.127297 15649 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:36:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.131807 15649 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:36:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.132064 15649 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:36:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.132390 15649 ts_tablet_manager.cc:610] Registered 0 tablets
20:36:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.132627 15649 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.001s
20:36:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.441844 15649 rpc_server.cc:307] RPC server started. Bound to: 127.14.177.131:41287
20:36:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.441959 15784 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.177.131:41287 every 8 connection(s)
20:36:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.444491 15649 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/data/info.pb
20:36:25.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.448382 15046 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15649
20:36:25.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.448844 15046 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster4462075032525047368/ts-2/wal/instance
20:36:25.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.515642 15786 heartbeater.cc:347] Connected to a master server at 127.14.177.189:42485
20:36:25.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.516110 15786 heartbeater.cc:464] Registering TS with master...
20:36:25.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.517488 15785 heartbeater.cc:347] Connected to a master server at 127.14.177.188:45045
20:36:25.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.517740 15786 heartbeater.cc:511] Master 127.14.177.189:42485 requested a full tablet report, sending...
20:36:25.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.518031 15785 heartbeater.cc:464] Registering TS with master...
20:36:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.519336 15789 heartbeater.cc:347] Connected to a master server at 127.14.177.190:36727
20:36:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.519416 15785 heartbeater.cc:511] Master 127.14.177.188:45045 requested a full tablet report, sending...
20:36:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.520001 15789 heartbeater.cc:464] Registering TS with master...
20:36:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.521466 15789 heartbeater.cc:511] Master 127.14.177.190:36727 requested a full tablet report, sending...
20:36:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.522379 15170 ts_manager.cc:194] Registered new tserver with Master: 6fe9fa4621124370a49453e468714be5 (127.14.177.131:41287)
20:36:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.525067 15252 ts_manager.cc:194] Registered new tserver with Master: 6fe9fa4621124370a49453e468714be5 (127.14.177.131:41287)
20:36:25.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.525954 15091 ts_manager.cc:194] Registered new tserver with Master: 6fe9fa4621124370a49453e468714be5 (127.14.177.131:41287)
20:36:25.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.528239 15252 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.177.131:58173
20:36:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:25.537693 15046 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:36:25.556 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:36:25.557 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:36:25.569 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.14.177.190"
port: 36727
}
bound_http_address {
host: "127.14.177.190"
port: 38487
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.14.177.189"
port: 42485
}
bound_http_address {
host: "127.14.177.189"
port: 42185
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.14.177.188"
port: 45045
}
bound_http_address {
host: "127.14.177.188"
port: 45641
}
}
}
20:36:25.576 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:36:25.578 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.14.177.129"
port: 37107
}
bound_http_address {
host: "127.14.177.129"
port: 39591
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.14.177.130"
port: 42175
}
bound_http_address {
host: "127.14.177.130"
port: 33695
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.14.177.131"
port: 41287
}
bound_http_address {
host: "127.14.177.131"
port: 38027
}
}
}
20:36:25.579 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:36:25.779 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:36:25.926 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskm_a55n/test-tmp
20:36:25.927 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:36:25.927 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskm_a55n/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:36:25.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:36:26.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:26.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:36:26.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:26.424835 15797 mini_oidc.cc:152] Starting JWKS server
20:36:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:26.432222 15797 webserver.cc:460] Webserver started at http://127.0.0.1:33773/ using document root <none> and password file <none>
20:36:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:26.432698 15797 mini_oidc.cc:189] Starting OIDC Discovery server
20:36:26.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:26.436883 15797 webserver.cc:460] Webserver started at http://127.0.0.1:43511/ using document root <none> and password file <none>
20:36:26.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:36:26.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/principal' for realm 'KRBTEST.COM',
20:36:26.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:36:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:26.533497 15785 heartbeater.cc:503] Master 127.14.177.188:45045 was elected leader, sending a full tablet report...
20:36:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:26 dist-test-slave-bt4h krb5kdc[15810](info): setting up network...
20:36:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:36:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:26 dist-test-slave-bt4h krb5kdc[15810](info): set up 2 sockets
20:36:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:26 dist-test-slave-bt4h krb5kdc[15810](info): commencing operation
20:36:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:36:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:27.619141 15797 mini_kdc.cc:121] Time spent starting KDC: real 1.134s user 0.003s sys 0.020s
20:36:27.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:36:27.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:36:27.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:36:27.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:36:27.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:36:27.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:36:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:36:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:36:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:36:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:36:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/test-user.keytab.
20:36:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/test-user.keytab.
20:36:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:27 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048587, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:36:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:36:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:36:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:36:27Z Disabled control of system clock
20:36:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.109.126@KRBTEST.COM; defaulting to no policy
20:36:27.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:36:27.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.109.126@KRBTEST.COM" created.
20:36:27.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:36:27.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.109.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.126.keytab.
20:36:27.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.109.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.126.keytab.
20:36:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:27.840041 15797 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data
20:36:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/logs
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/info.pb
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.126:45011
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.126
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.126.keytab
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.126
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:36:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:36:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:36:28.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.240259 15823 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:28.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.240929 15823 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:28.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.241523 15823 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.275203 15823 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:36:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.275439 15823 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.275703 15823 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.276135 15823 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.276499 15823 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.276834 15823 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.277066 15823 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33773/jwks.json
20:36:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:28.315361 15823 master_runner.cc:386] Master server non-default flags:
20:36:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.126.keytab
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.126
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.126:45011
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/info.pb
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.126
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/logs
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:28.316927 15823 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:28.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:28.318112 15823 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:28.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.362306 15831 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:29.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.364323 15830 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:28.365973 15833 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:29.839588 15832 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1478 milliseconds
20:36:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.839711 15823 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:36:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.843626 15823 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.847160 15823 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.848609 15823 hybrid_clock.cc:648] HybridClock initialized: now 1786048589848538 us; error 89 us; skew 500 ppm
20:36:29.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.849534 15823 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:29.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:29 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048589, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:29.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.857710 15823 init.cc:377] Logged in from keytab as kudu/127.15.109.126@KRBTEST.COM (short username kudu)
20:36:29.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.867094 15823 webserver.cc:460] Webserver started at http://127.15.109.126:38449/ using document root <none> and password file <none>
20:36:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.868041 15823 fs_manager.cc:359] Metadata directory not provided
20:36:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.868221 15823 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.868597 15823 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.873878 15823 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/instance:
20:36:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "51f687f4e3e74d099a4d3879c942a14f"
20:36:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:29 on dist-test-slave-bt4h"
20:36:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.875043 15823 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal/instance:
20:36:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "51f687f4e3e74d099a4d3879c942a14f"
20:36:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:29 on dist-test-slave-bt4h"
20:36:29.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.883783 15823 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
20:36:29.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.891265 15841 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:29.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.892459 15823 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.004s sys 0.001s
20:36:29.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.892930 15823 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:29.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "51f687f4e3e74d099a4d3879c942a14f"
20:36:29.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:29 on dist-test-slave-bt4h"
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.893401 15823 fs_report.cc:383] FS layout report
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/data
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.935302 15823 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:29.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.941087 15823 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:29.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:29.941627 15823 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.099637 15823 rpc_server.cc:307] RPC server started. Bound to: 127.15.109.126:45011
20:36:30.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.099774 15894 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.109.126:45011 every 8 connection(s)
20:36:30.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.103284 15823 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/info.pb
20:36:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.109400 15895 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:36:30.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.110337 15797 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15823
20:36:30.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.110792 15797 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal/instance
20:36:30.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.109.125@KRBTEST.COM; defaulting to no policy
20:36:30.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.129050 15895 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } has no permanent_uuid. Determining permanent_uuid...
20:36:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:36:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.109.125@KRBTEST.COM" created.
20:36:30.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:36:30.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.109.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.125.keytab.
20:36:30.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.109.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.125.keytab.
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.173072 15797 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/logs
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/info.pb
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.125:42037
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.125
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.125.keytab
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.125
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:36:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:30 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048589, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.126@KRBTEST.COM for kudu/127.15.109.126@KRBTEST.COM
20:36:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.193305 15895 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } has no permanent_uuid. Determining permanent_uuid...
20:36:30.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.196486 15843 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:30.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.201427 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.125:42037: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:30.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.250931 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } attempt: 1
20:36:30.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.254967 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.125:42037: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:30.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.323396 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } attempt: 2
20:36:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.327962 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.125:42037: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.419425 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } attempt: 3
20:36:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.423563 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.125:42037: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:30.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.539533 15901 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.540081 15901 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.540664 15901 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.566948 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } attempt: 4
20:36:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.570690 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.125:42037: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:30.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.576743 15901 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:36:30.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.577019 15901 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:30.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.577481 15901 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:30.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.577808 15901 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.578146 15901 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.578465 15901 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.578752 15901 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33773/jwks.json
20:36:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.616173 15901 master_runner.cc:386] Master server non-default flags:
20:36:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data
20:36:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.125.keytab
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.125
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.125:42037
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/info.pb
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.125
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/logs
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:30.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.617712 15901 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.619174 15901 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.642354 15910 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:30.870075 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } attempt: 5
20:36:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.873984 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.125:42037: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.421612 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } attempt: 6
20:36:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:31.433223 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.125:42037: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.642625 15912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:30.642726 15909 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:31.908399 15911 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:36:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.908542 15901 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:36:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.912560 15901 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.915876 15901 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:31.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.917332 15901 hybrid_clock.cc:648] HybridClock initialized: now 1786048591917295 us; error 42 us; skew 500 ppm
20:36:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.918289 15901 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:31.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:31 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048591, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:31.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.925716 15901 init.cc:377] Logged in from keytab as kudu/127.15.109.125@KRBTEST.COM (short username kudu)
20:36:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.932991 15901 webserver.cc:460] Webserver started at http://127.15.109.125:46575/ using document root <none> and password file <none>
20:36:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.933911 15901 fs_manager.cc:359] Metadata directory not provided
20:36:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.934104 15901 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.934626 15901 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.939831 15901 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/instance:
20:36:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b7abb4eab244dc930b79222591a7a0"
20:36:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:31 on dist-test-slave-bt4h"
20:36:31.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.940943 15901 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal/instance:
20:36:31.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b7abb4eab244dc930b79222591a7a0"
20:36:31.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:31 on dist-test-slave-bt4h"
20:36:31.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.948817 15901 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.007s
20:36:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.955299 15924 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:31.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.956423 15901 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.956717 15901 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b7abb4eab244dc930b79222591a7a0"
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:31 on dist-test-slave-bt4h"
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:31.957038 15901 fs_report.cc:383] FS layout report
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/data
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:31.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.004339 15901 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.011150 15901 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.011598 15901 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.115068 15901 rpc_server.cc:307] RPC server started. Bound to: 127.15.109.125:42037
20:36:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.115742 15975 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.109.125:42037 every 8 connection(s)
20:36:32.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.117818 15901 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/info.pb
20:36:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.124794 15976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:36:32.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.126435 15797 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15901
20:36:32.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.126978 15797 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal/instance
20:36:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.109.124@KRBTEST.COM; defaulting to no policy
20:36:32.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:36:32.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.109.124@KRBTEST.COM" created.
20:36:32.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.142511 15976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } has no permanent_uuid. Determining permanent_uuid...
20:36:32.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:36:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.109.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.124.keytab.
20:36:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.109.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.124.keytab.
20:36:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:32 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048591, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.125@KRBTEST.COM for kudu/127.15.109.126@KRBTEST.COM
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.175843 15797 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/logs
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/info.pb
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.124:45285
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.124
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.124.keytab
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.124
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:36:32.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.187104 15976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } has no permanent_uuid. Determining permanent_uuid...
20:36:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:32 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048591, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.125@KRBTEST.COM for kudu/127.15.109.125@KRBTEST.COM
20:36:32.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.219472 15976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } has no permanent_uuid. Determining permanent_uuid...
20:36:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.222643 15927 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.229818 15976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.279299 15976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 1
20:36:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.283617 15976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.351984 15976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 2
20:36:32.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.356014 15976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.447395 15976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 3
20:36:32.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.451416 15976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.493829 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } attempt: 7
20:36:32.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:32 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048589, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.126@KRBTEST.COM for kudu/127.15.109.125@KRBTEST.COM
20:36:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.522317 15895 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } has no permanent_uuid. Determining permanent_uuid...
20:36:32.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.530417 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.561213 15982 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:32.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.561836 15982 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:32.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.563228 15982 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:32.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.588894 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 1
20:36:32.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.593006 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.594852 15976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 4
20:36:32.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.599848 15976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.603097 15982 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:36:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.603389 15982 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.603677 15982 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:32.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.603873 15982 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:32.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.604068 15982 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:32.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.604262 15982 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:32.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.604400 15982 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33773/jwks.json
20:36:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.641351 15982 master_runner.cc:386] Master server non-default flags:
20:36:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data
20:36:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.124.keytab
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.124
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.124:45285
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/info.pb
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.124
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/logs
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.642473 15982 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:32.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.643569 15982 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:32.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.667052 15993 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:32.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.674384 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 2
20:36:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.678695 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.764148 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 3
20:36:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.771215 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.899286 15976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 5
20:36:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.903911 15976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:32.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.911674 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 4
20:36:32.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.916495 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.199905 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 5
20:36:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:33.204856 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:33.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.451308 15976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 6
20:36:33.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:33.460644 15976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:33.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.757292 15895 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } attempt: 6
20:36:33.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:33.764086 15895 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.109.124:45285: Network error: Client connection negotiation failed: client connection to 127.15.109.124:45285: connect: Connection refused (error 111)
20:36:33.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.668705 15996 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:33.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:32.668905 15994 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:33.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:32.669833 15982 server_base.cc:1034] running on GCE node
20:36:33.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.943106 15982 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:33.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.946246 15982 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:33.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.947736 15982 hybrid_clock.cc:648] HybridClock initialized: now 1786048593947679 us; error 87 us; skew 500 ppm
20:36:33.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.948632 15982 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:33.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:33 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048593, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:33.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.955168 15982 init.cc:377] Logged in from keytab as kudu/127.15.109.124@KRBTEST.COM (short username kudu)
20:36:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.961732 15982 webserver.cc:460] Webserver started at http://127.15.109.124:42681/ using document root <none> and password file <none>
20:36:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.962716 15982 fs_manager.cc:359] Metadata directory not provided
20:36:33.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.963016 15982 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:33.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.963485 15982 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:36:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.969626 15982 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/instance:
20:36:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b87de7591ec4c4e99e2a3ba0a36257a"
20:36:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:33 on dist-test-slave-bt4h"
20:36:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.971184 15982 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal/instance:
20:36:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b87de7591ec4c4e99e2a3ba0a36257a"
20:36:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:33 on dist-test-slave-bt4h"
20:36:33.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.982723 15982 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.006s sys 0.005s
20:36:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.990762 16009 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:33.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.992110 15982 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
20:36:33.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.992520 15982 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:33.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b87de7591ec4c4e99e2a3ba0a36257a"
20:36:33.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:33 on dist-test-slave-bt4h"
20:36:33.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:33.992950 15982 fs_report.cc:383] FS layout report
20:36:33.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:33.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/data
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.043605 15982 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.049538 15982 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:34.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.050289 15982 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.151274 15982 rpc_server.cc:307] RPC server started. Bound to: 127.15.109.124:45285
20:36:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.151403 16061 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.109.124:45285 every 8 connection(s)
20:36:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.155189 15982 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/info.pb
20:36:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.162828 16062 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:36:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.163205 15797 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 15982
20:36:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.163738 15797 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal/instance
20:36:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.181514 16062 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } has no permanent_uuid. Determining permanent_uuid...
20:36:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:34 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048587, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.109.124@KRBTEST.COM
20:36:34.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:34 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048593, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.124@KRBTEST.COM for kudu/127.15.109.126@KRBTEST.COM
20:36:34.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.226081 16062 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } has no permanent_uuid. Determining permanent_uuid...
20:36:34.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:34 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048593, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.124@KRBTEST.COM for kudu/127.15.109.125@KRBTEST.COM
20:36:34.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:34 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048587, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.109.125@KRBTEST.COM
20:36:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.257941 16062 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } has no permanent_uuid. Determining permanent_uuid...
20:36:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:34 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048587, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.109.126@KRBTEST.COM
20:36:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:34 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048593, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.124@KRBTEST.COM for kudu/127.15.109.124@KRBTEST.COM
20:36:34.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.294660 15797 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:36:34.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.298102 16062 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: Bootstrap starting.
20:36:34.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:34.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.304647 16062 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: Neither blocks nor log segments found. Creating new log.
20:36:34.306 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:36:34.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.306896 16062 log.cc:824] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: Log is configured to *not* fsync() on all Append() calls
20:36:34.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.312271 16062 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: No bootstrap required, opened a new log
20:36:34.320 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:36:34.322 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:36:34.325 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.109.126"
port: 45011
}
bound_http_address {
host: "127.15.109.126"
port: 38449
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.109.125"
port: 42037
}
bound_http_address {
host: "127.15.109.125"
port: 46575
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.109.124"
port: 45285
}
bound_http_address {
host: "127.15.109.124"
port: 42681
}
}
}
20:36:34.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:36:34.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:34.326 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:36:34.328 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:36:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.347662 16062 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:34.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.348706 16062 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:36:34.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.349068 16062 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5b87de7591ec4c4e99e2a3ba0a36257a, State: Initialized, Role: FOLLOWER
20:36:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.350442 16062 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:34.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.354805 16073 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } } }
20:36:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.355787 16073 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [sys.catalog]: This master's current role is: FOLLOWER
20:36:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.356220 16062 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [sys.catalog]: configured and running, proceeding with master startup.
20:36:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:34 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048594, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:36:34.367 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:34.370 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc
20:36:34.380 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:36:34.387 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc
20:36:34.389 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:36:34.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.394161 16084 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:36:34.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.395756 16073 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:36:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.396271 16073 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:34.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.400347 16073 leader_election.cc:288] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011), 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037)
20:36:34.404 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:36:34.406 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:36:34.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:36:34.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.398222 15982 master_runner.cc:186] Error getting master registration for 127.15.109.126:45011: OK, instance_id { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" instance_seqno: 1786048590059806 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:36:34.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.430238 15982 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:36:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:34 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048594, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:36:34.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:34.449 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc
20:36:34.450 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:36:34.451 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.109.126:45011
20:36:34.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:36:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.454178 15797 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15823
20:36:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.478058 16010 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.15.109.126:45011 (error 108)
20:36:34.482 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:34.482 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.109.124:45285
20:36:34.482 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:36:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.482771 16010 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011): Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111)
20:36:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.483075 15797 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15982
20:36:34.508 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:34.508 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.109.125:42037
20:36:34.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:36:34.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.509940 15797 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15901
20:36:34.530 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:34.531 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.109.126:45011
20:36:34.533 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.534731 15797 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/logs
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/info.pb
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.126:45011
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.126
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38449
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.126.keytab
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.126
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:36:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.870687 16089 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.871243 16089 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:34.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.871838 16089 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.907860 16089 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:36:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.908099 16089 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.908447 16089 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.908674 16089 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.908864 16089 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.909055 16089 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.909189 16089 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33773/jwks.json
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.949262 16089 master_runner.cc:386] Master server non-default flags:
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.126.keytab
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.126
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.126:45011
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/info.pb
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.126
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38449
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/logs
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.950582 16089 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:34.951747 16089 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.970906 16096 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.971580 16095 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:34.970930 16098 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:36.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.274667 16097 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1303 milliseconds
20:36:36.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.274789 16089 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:36:36.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.276122 16089 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:36.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.279194 16089 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:36.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.280735 16089 hybrid_clock.cc:648] HybridClock initialized: now 1786048596280701 us; error 48 us; skew 500 ppm
20:36:36.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.281646 16089 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:36 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048596, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.292418 16089 init.cc:377] Logged in from keytab as kudu/127.15.109.126@KRBTEST.COM (short username kudu)
20:36:36.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.298285 16089 webserver.cc:460] Webserver started at http://127.15.109.126:38449/ using document root <none> and password file <none>
20:36:36.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.299464 16089 fs_manager.cc:359] Metadata directory not provided
20:36:36.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.299753 16089 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:36.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.308530 16089 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
20:36:36.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.312871 16106 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:36.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.314090 16089 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.314455 16089 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "51f687f4e3e74d099a4d3879c942a14f"
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:29 on dist-test-slave-bt4h"
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.317096 16089 fs_report.cc:383] FS layout report
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/data
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:36.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:36.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:36.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:36.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:36.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:36.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:36.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.353529 16089 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:36.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.359506 16089 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:36.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.359941 16089 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:36.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.438951 16089 rpc_server.cc:307] RPC server started. Bound to: 127.15.109.126:45011
20:36:36.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.439080 16159 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.109.126:45011 every 8 connection(s)
20:36:36.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.441845 16089 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/data/info.pb
20:36:36.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.447407 15797 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 16089
20:36:36.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:36.448 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.109.124:45285
20:36:36.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:36:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.450659 15797 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data
20:36:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:36.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:36.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:36.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/logs
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/info.pb
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.124:45285
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.124
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42681
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.124.keytab
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.124
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:36:36.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.456406 16160 sys_catalog.cc:263] Verifying existing consensus state
20:36:36.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 20:36:36.457312 16160 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:36:36.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.457787 16089 master.cc:537] Master@127.15.109.126:45011 shutting down...
20:36:36.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.515852 16089 master.cc:552] Master@127.15.109.126:45011 shutdown complete.
20:36:36.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:36:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.779749 16162 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.780381 16162 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:36.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.781332 16162 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.815541 16162 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:36:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.815784 16162 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.816145 16162 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.816357 16162 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.816550 16162 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.816732 16162 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.816871 16162 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33773/jwks.json
20:36:36.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.853600 16162 master_runner.cc:386] Master server non-default flags:
20:36:36.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:36.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:36.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:36.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:36.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.124.keytab
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.124
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.124:45285
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/info.pb
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.124
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42681
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/logs
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.854972 16162 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:36.856338 16162 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:36.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.875999 16171 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:38.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.876305 16168 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:38.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:36.876006 16169 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.197572 16170 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1319 milliseconds
20:36:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.197691 16162 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:36:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.199097 16162 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.201615 16162 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.203006 16162 hybrid_clock.cc:648] HybridClock initialized: now 1786048598202974 us; error 41 us; skew 500 ppm
20:36:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.203917 16162 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:38.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:38 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048598, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:38.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.209913 16162 init.cc:377] Logged in from keytab as kudu/127.15.109.124@KRBTEST.COM (short username kudu)
20:36:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.216187 16162 webserver.cc:460] Webserver started at http://127.15.109.124:42681/ using document root <none> and password file <none>
20:36:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.217178 16162 fs_manager.cc:359] Metadata directory not provided
20:36:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.217466 16162 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.227310 16162 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
20:36:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.232131 16180 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.233459 16162 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.233844 16162 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b87de7591ec4c4e99e2a3ba0a36257a"
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:33 on dist-test-slave-bt4h"
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.236219 16162 fs_report.cc:383] FS layout report
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/wal
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/data
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.297187 16162 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:38.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.303478 16162 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.303978 16162 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.377061 16162 rpc_server.cc:307] RPC server started. Bound to: 127.15.109.124:45285
20:36:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.377197 16231 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.109.124:45285 every 8 connection(s)
20:36:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.380302 16162 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-2/data/info.pb
20:36:38.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.386291 15797 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 16162
20:36:38.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:38.387 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.109.125:42037
20:36:38.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:36:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.389539 15797 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/logs
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/info.pb
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.125:42037
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.125
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46575
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.125.keytab
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.125
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:36:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.393128 16232 sys_catalog.cc:263] Verifying existing consensus state
20:36:38.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.400899 16232 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: Bootstrap starting.
20:36:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.435895 16232 log.cc:824] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: Log is configured to *not* fsync() on all Append() calls
20:36:38.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.440436 16232 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:36:38.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.441195 16232 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: Bootstrap complete.
20:36:38.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.466280 16232 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:38.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.468837 16232 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:36:38.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.469169 16232 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5b87de7591ec4c4e99e2a3ba0a36257a, State: Initialized, Role: FOLLOWER
20:36:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.470129 16232 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:38.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.472798 16236 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } } }
20:36:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.474381 16232 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [sys.catalog]: configured and running, proceeding with master startup.
20:36:38.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.474658 16236 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [sys.catalog]: This master's current role is: FOLLOWER
20:36:38.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.518646 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:38.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.519125 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.508749 16162 master_runner.cc:186] Error getting master registration for 127.15.109.126:45011: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111),
20:36:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.536190 16162 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:36:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.559406 16186 tablet.cc:2369] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:36:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.725042 16236 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:36:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.725515 16236 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:38.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.729713 16236 leader_election.cc:288] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011), 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037)
20:36:38.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.732026 16181 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111)
20:36:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.745925 16181 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011): Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111)
20:36:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.746788 16182 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037): Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.747182 16182 leader_election.cc:302] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b87de7591ec4c4e99e2a3ba0a36257a; no voters: 51f687f4e3e74d099a4d3879c942a14f, 65b7abb4eab244dc930b79222591a7a0
20:36:38.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.747967 16236 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:36:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.770349 16234 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:36:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.771204 16234 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:36:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.772258 16234 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:36:38.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.810382 16234 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:36:38.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.810732 16234 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:36:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.811187 16234 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:36:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.811410 16234 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:36:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.811597 16234 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:36:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.811785 16234 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:36:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.811923 16234 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33773/jwks.json
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.848585 16234 master_runner.cc:386] Master server non-default flags:
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.109.84:44109
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.109.126:45011,127.15.109.125:42037,127.15.109.124:45285
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/krb5kdc/kudu_127.15.109.125.keytab
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.109.125
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.109.125:42037
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33773/jwks.json
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:36:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/info.pb
20:36:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.109.125
20:36:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46575
20:36:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:36:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:36:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:36:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:36:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:36:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/logs
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:37:47 UTC on ccycloud.quasar-yousdc.root.comops.site
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571031
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.849725 16234 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:36:38.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:38.850771 16234 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:36:38.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.870411 16260 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:39.525192 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:39.525630 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:40.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.360054 16263 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:36:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.361313 16263 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:40.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.372117 16263 leader_election.cc:288] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011), 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037)
20:36:40.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:40.390928 16181 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011): Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111)
20:36:40.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:40.397156 16182 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037): Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:40.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.397634 16182 leader_election.cc:302] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b87de7591ec4c4e99e2a3ba0a36257a; no voters: 51f687f4e3e74d099a4d3879c942a14f, 65b7abb4eab244dc930b79222591a7a0
20:36:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.398626 16263 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:36:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:40.529978 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:40.531235 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:40.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.874009 16257 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:40.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:38.870504 16258 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:36:40.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:40.554486 16259 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1681 milliseconds
20:36:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.554665 16234 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:36:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.556209 16234 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:36:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.559437 16234 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:36:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.561028 16234 hybrid_clock.cc:648] HybridClock initialized: now 1786048600560990 us; error 44 us; skew 500 ppm
20:36:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.562279 16234 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:36:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:40 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048600, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.569653 16234 init.cc:377] Logged in from keytab as kudu/127.15.109.125@KRBTEST.COM (short username kudu)
20:36:40.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.576421 16234 webserver.cc:460] Webserver started at http://127.15.109.125:46575/ using document root <none> and password file <none>
20:36:40.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.577666 16234 fs_manager.cc:359] Metadata directory not provided
20:36:40.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.577960 16234 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:36:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.588696 16234 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.004s
20:36:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.594216 16273 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:36:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.595499 16234 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.595793 16234 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data,/tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b7abb4eab244dc930b79222591a7a0"
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:36:31 on dist-test-slave-bt4h"
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.598059 16234 fs_report.cc:383] FS layout report
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/data
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:36:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:36:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:36:40.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.645630 16234 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:36:40.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.651698 16234 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:36:40.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.652333 16234 kserver.cc:163] Server-wide thread pool size limit: 3276
20:36:40.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.743113 16234 rpc_server.cc:307] RPC server started. Bound to: 127.15.109.125:42037
20:36:40.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.743232 16326 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.109.125:42037 every 8 connection(s)
20:36:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.745934 16234 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/data/info.pb
20:36:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.748772 15797 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu as pid 16234
20:36:40.749 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:36:40.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.783993 16327 sys_catalog.cc:263] Verifying existing consensus state
20:36:40.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 20:36:40.787533 16327 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:36:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.788046 16234 master.cc:537] Master@127.15.109.125:42037 shutting down...
20:36:40.846 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:40.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:40.924516 16234 master.cc:552] Master@127.15.109.125:42037 shutdown complete.
20:36:40.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskm_a55n/test-tmp/mini-kudu-cluster8424356560837364954/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:36:40.954 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.109.124:45285(127.15.109.124:45285)] connecting to peer
20:36:40.956 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.020 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.15.109.124:45285(127.15.109.124:45285)] Successfully connected to peer
20:36:41.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.028 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.120 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:36:41.141 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.109.124
20:36:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:41 dist-test-slave-bt4h krb5kdc[15810](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048594, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.109.124@KRBTEST.COM
20:36:41.401 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd2c44c84, L:/127.0.0.1:54976 - R:/127.15.109.124:45285] using SASL/GSSAPI
20:36:41.418 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:41.436 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:41.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.441 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.441 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.443 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.444 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.448 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:41.455 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:41.457 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.459 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.460 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.463 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:41.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:41.477 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.481 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.483 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:41.495 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:41.496 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.500 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.503 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:41.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:41.534765 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:41.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:41.535030 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:41.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:41.537 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.539 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.541 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.542 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.543 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:41.575 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:41.577 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.580 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.583 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:41.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:41.657 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.659 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.659 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.660 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.663 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:41.875 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:41.876 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:41.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.880 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:41.880 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:41.881 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:42.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:42.377 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:42.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:42.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:42.380 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:42.382 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:42.382 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:42.383 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:42.383 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:42.538069 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:42.538336 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:42.915 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:42.916 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:42.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:42.919 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:42.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:42.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:42.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:42.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:42.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:43.538838 16162 master_runner.cc:186] Error getting master registration for 127.15.109.126:45011: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111),
20:36:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:43.539289 16162 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:36:43.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:43.541364 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:43.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:43.541589 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:43.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:43.736073 16338 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:36:43.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:43.736577 16338 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:43.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:43.738158 16338 leader_election.cc:288] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011), 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037)
20:36:43.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:43.739300 16182 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:36:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:43.743157 16182 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037): Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:43.743752 16181 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011): Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111)
20:36:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:43.744118 16181 leader_election.cc:302] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b87de7591ec4c4e99e2a3ba0a36257a; no voters: 51f687f4e3e74d099a4d3879c942a14f, 65b7abb4eab244dc930b79222591a7a0
20:36:43.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:43.744794 16338 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:36:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:44.544523 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:44.544777 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:44.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:44.957 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:44.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:44.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:44.959 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:44.962 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:44.962 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:44.962 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:44.962 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:45.547819 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:45.548076 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:45.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:45.931540 16342 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:36:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:45.931941 16342 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:45.933357 16342 leader_election.cc:288] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011), 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037)
20:36:45.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:45.937587 16181 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011): Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111)
20:36:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:45.938009 16182 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037): Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:45.938293 16182 leader_election.cc:302] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b87de7591ec4c4e99e2a3ba0a36257a; no voters: 51f687f4e3e74d099a4d3879c942a14f, 65b7abb4eab244dc930b79222591a7a0
20:36:45.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:45.938851 16342 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:36:46.095 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:46.097 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:46.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:46.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:46.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:46.099 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:46.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:46.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:46.101 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:46.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:46.551410 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:46.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:46.551654 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:47.555609 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:47.556012 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:48 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048608, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:48.214010 16176 init.cc:295] Successfully reacquired a new kerberos TGT
20:36:48.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:48.542644 16162 master_runner.cc:186] Error getting master registration for 127.15.109.126:45011: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111),
20:36:48.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:48.543048 16162 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:36:48.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:48.559075 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:48.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:48.559355 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:49.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:49.357 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:49.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:49.360 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:49.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:49.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:49.360 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:49.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:49.363 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:49.562222 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:49.562511 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:50.565399 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:50.565685 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:51.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:51.384542 16354 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:36:51.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:51.385104 16354 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:36:51.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:51.386965 16354 leader_election.cc:288] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011), 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037)
20:36:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:51.389396 16181 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:36:51.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:51.392614 16182 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037): Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:36:51.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:51.393404 16181 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011): Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111)
20:36:51.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:51.393828 16181 leader_election.cc:302] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b87de7591ec4c4e99e2a3ba0a36257a; no voters: 51f687f4e3e74d099a4d3879c942a14f, 65b7abb4eab244dc930b79222591a7a0
20:36:51.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:51.394619 16354 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:36:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:51.569173 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:51.569447 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:52.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:52.477 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:52.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:52.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:52.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:52.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:52.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:52.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:52.481 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:52.572513 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:52.572851 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:53.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:53.545714 16162 master_runner.cc:186] Error getting master registration for 127.15.109.126:45011: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111),
20:36:53.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:53.546084 16162 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:36:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:53.576087 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:53.576491 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:54.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:54.579382 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:54.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:54.579706 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:55.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:55.582691 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:55.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:55.582995 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:55.775 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:55.777 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:55.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:55.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:55.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:55.780 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:55.780 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:55.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:55.781 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:56.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:56.297573 15285 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2000 after lost signal to thread 15213
20:36:56.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:56.298174 15285 debug-util.cc:398] Leaking SignalData structure 0x7b08000baf60 after lost signal to thread 15288
20:36:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:56.585944 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:56.586232 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:57.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:57.589891 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:57.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:57.590226 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:58.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:36:58 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048618, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:36:58.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:58.216606 16176 init.cc:295] Successfully reacquired a new kerberos TGT
20:36:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:58.549480 16162 master_runner.cc:186] Error getting master registration for 127.15.109.126:45011: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111),
20:36:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:36:58.549916 16162 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:36:58.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:58.593245 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:58.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:58.593562 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:36:59.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:36:59.376 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:36:59.379 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:59.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:59.379 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:59.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:59.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:36:59.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:36:59.381 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:36:59.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:59.597047 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:36:59.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:36:59.597333 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:00.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:00.600536 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:00.600845 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:01.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:01.603899 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:01.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:01.604195 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:01.932451 16372 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:37:01.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:01.932957 16372 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "51f687f4e3e74d099a4d3879c942a14f" member_type: VOTER last_known_addr { host: "127.15.109.126" port: 45011 } } peers { permanent_uuid: "65b7abb4eab244dc930b79222591a7a0" member_type: VOTER last_known_addr { host: "127.15.109.125" port: 42037 } } peers { permanent_uuid: "5b87de7591ec4c4e99e2a3ba0a36257a" member_type: VOTER last_known_addr { host: "127.15.109.124" port: 45285 } }
20:37:01.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:01.934909 16372 leader_election.cc:288] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011), 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037)
20:37:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:01.936802 16181 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:37:01.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:01.940248 16182 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65b7abb4eab244dc930b79222591a7a0 (127.15.109.125:42037): Network error: Client connection negotiation failed: client connection to 127.15.109.125:42037: connect: Connection refused (error 111)
20:37:01.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:01.940743 16181 leader_election.cc:334] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 51f687f4e3e74d099a4d3879c942a14f (127.15.109.126:45011): Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111)
20:37:01.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:01.941023 16181 leader_election.cc:302] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b87de7591ec4c4e99e2a3ba0a36257a; no voters: 51f687f4e3e74d099a4d3879c942a14f, 65b7abb4eab244dc930b79222591a7a0
20:37:01.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:01.941610 16372 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:37:02.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:02.607498 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:02.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:02.607839 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:03.395 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:37:03.397 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:37:03.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:03.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:03.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:03.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:03.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:03.399 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:03.401 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:37:03.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:03.552757 16162 master_runner.cc:186] Error getting master registration for 127.15.109.126:45011: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111),
20:37:03.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:03.553179 16162 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:37:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:03.611160 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:03.611483 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:04.614435 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:04.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:04.614795 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:05.300119 15635 debug-util.cc:398] Leaking SignalData structure 0x7b08000becc0 after lost signal to thread 15507
20:37:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:05.300942 15635 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbb20 after lost signal to thread 15638
20:37:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:05.617758 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:05.618049 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:06.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:06.621220 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:06.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:06.621520 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:06.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:37:06.857 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:37:06.860 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:06.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:06.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:06.862 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:06.863 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:06.863 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:06.865 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:37:07.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:07.625102 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:07.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:07.625517 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:08.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:37:08 dist-test-slave-bt4h krb5kdc[15810](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048628, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:37:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:08.218972 16176 init.cc:295] Successfully reacquired a new kerberos TGT
20:37:08.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:08.556409 16162 master_runner.cc:186] Error getting master registration for 127.15.109.126:45011: Network error: Client connection negotiation failed: client connection to 127.15.109.126:45011: connect: Connection refused (error 111),
20:37:08.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:08.556826 16162 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:37:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:08.628466 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:08.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:08.628770 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:09.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:09.632390 16247 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5b87de7591ec4c4e99e2a3ba0a36257a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:09.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:37:09.632714 16247 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:10.275 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.126:45011(127.15.109.126:45011)] connecting to peer
20:37:10.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:10.277 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.126:45011(127.15.109.126:45011)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:10.277 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.109.125:42037(127.15.109.125:42037)] connecting to peer
20:37:10.277 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.126:45011: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011
20:37:10.279 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:10.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.109.125:42037(127.15.109.125:42037)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:10.280 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.109.125:42037: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
20:37:10.281 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037; will retry
20:37:10.286 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=20, TimeoutTracker(timeout=30000, elapsed=29528), Traces: [0ms] refreshing cache from master, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [267ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [271ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [654ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [659ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [682ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [684ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [689ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [700ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [701ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [703ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [723ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [740ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [742ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [743ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [780ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [782ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [784ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [816ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [823ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [900ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [901ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [903ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1115ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1123ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1623ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1625ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2155ms] refreshing cache from master, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [2157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2161ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [2162ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4196ms] refreshing cache from master, [4196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [4197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [4198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [4200ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4202ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [4203ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [4203ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [5336ms] refreshing cache from master, [5336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [5338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@1871542623(state=PENDING, result=null, callback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534), errback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534)))
20:37:10.291 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.109.124:45285(127.15.109.124:45285)] disconnecting while in state READY
20:37:10.292 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.109.126:45011(127.15.109.126:45011)] disconnecting while in state TERMINATED
20:37:10.292 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.109.125:42037(127.15.109.125:42037)] disconnecting while in state TERMINATED
20:37:10.297 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:37:10.299 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.15.109.124:45285(127.15.109.124:45285)] handling channelInactive
20:37:10.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.109.124:45285(127.15.109.124:45285)] cleaning up while in state READY due to: connection closed
20:37:10.301 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:37:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:10.305642 15797 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 16162
20:37:11.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:37:11Z chronyd exiting
20:37:12.525 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:37:12.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:12.527432 15046 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15337
20:37:12.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:12.559118 15046 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15506
20:37:12.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:12.587388 15046 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15649
20:37:12.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:12.613510 15046 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15058
20:37:12.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:12.647022 15046 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15131
20:37:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:37:12.678324 15046 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskm_a55n/build/tsan/bin/kudu with pid 15208
20:37:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:37:13Z chronyd exiting
20:37:14.871 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=20, TimeoutTracker(timeout=30000, elapsed=29528), Traces: [0ms] refreshing cache from master, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [267ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [271ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [654ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [659ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [682ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [684ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [689ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [700ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [701ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [703ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [723ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [740ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [742ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [743ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [780ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [782ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [784ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [816ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [823ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [900ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [901ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [903ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1115ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1123ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1623ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1625ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2155ms] refreshing cache from master, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [2157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2161ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [2162ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4196ms] refreshing cache from master, [4196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [4197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [4198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [4200ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4202ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [4203ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [4203ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [5336ms] refreshing cache from master, [5336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [5338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@1871542623(state=PENDING, result=null, callback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534), errback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.109.125:42037
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=20, TimeoutTracker(timeout=30000, elapsed=29528), Traces: [0ms] refreshing cache from master, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [267ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [271ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [654ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [659ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [682ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [684ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [689ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [700ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [701ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [703ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [723ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [740ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [742ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [743ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [780ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [782ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [784ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [816ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [823ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [900ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [901ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [903ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1115ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1123ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1623ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1625ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2155ms] refreshing cache from master, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [2157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2161ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [2162ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4196ms] refreshing cache from master, [4196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [4197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [4198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [4200ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4202ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [4203ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [4203ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [5336ms] refreshing cache from master, [5336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [5338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@1871542623(state=PENDING, result=null, callback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534), errback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.109.125:42037
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
20:37:15.275 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:37:15.275 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=20, TimeoutTracker(timeout=30000, elapsed=29528), Traces: [0ms] refreshing cache from master, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [267ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [271ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [654ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [659ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [682ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [684ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [689ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [700ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [701ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [703ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [723ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [740ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [742ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [743ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [780ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [782ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [784ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [816ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [823ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [900ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [901ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [903ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1115ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1123ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1623ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1625ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2155ms] refreshing cache from master, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [2157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2161ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [2162ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4196ms] refreshing cache from master, [4196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [4197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [4198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [4200ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4202ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [4203ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [4203ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [5336ms] refreshing cache from master, [5336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [5338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@1871542623(state=PENDING, result=null, callback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534), errback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.109.125:42037
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=20, TimeoutTracker(timeout=30000, elapsed=29528), Traces: [0ms] refreshing cache from master, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [267ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [271ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [654ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [659ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [682ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [684ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [689ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [700ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [701ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [703ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [720ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [723ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [740ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [742ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [743ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [780ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [782ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [784ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [816ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [821ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [823ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [900ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [901ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [903ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1115ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1123ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [1623ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [1625ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2155ms] refreshing cache from master, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [2156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [2157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [2160ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [2161ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [2162ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4196ms] refreshing cache from master, [4196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [4197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [4198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [4200ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.125:42037: Network error: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037, [4202ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.124:45285: OK, [4203ms] Sub RPC ConnectToMaster: received response from server master-127.15.109.126:45011: Network error: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [4203ms] delaying RPC due to: Service unavailable: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011, [5336ms] refreshing cache from master, [5336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.126:45011, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.124:45285, [5338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.109.125:42037, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@1871542623(state=PENDING, result=null, callback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534), errback=(continuation of Deferred@1541798228 after retry RPC after error@335348793) -> (continuation of Deferred@771194746 after retry RPC after error@691317786) -> (continuation of Deferred@1567612721 after retry RPC after error@650691689) -> (continuation of Deferred@819094373 after retry RPC after error@1007308057) -> (continuation of Deferred@532651006 after retry RPC after error@2134175724) -> (continuation of Deferred@1549693476 after retry RPC after error@852663859) -> (continuation of Deferred@1372685308 after retry RPC after error@163732901) -> (continuation of Deferred@648204779 after retry RPC after error@11005962) -> (continuation of Deferred@1563712897 after retry RPC after error@1182797047) -> (continuation of Deferred@1990045765 after retry RPC after error@1233463612) -> (continuation of Deferred@916635096 after retry RPC after error@1887692879) -> (continuation of Deferred@1804038530 after retry RPC after error@195782919) -> (continuation of Deferred@683956695 after retry RPC after error@1206521104) -> (continuation of Deferred@389119758 after retry RPC after error@1199755198) -> (continuation of Deferred@1300376696 after retry RPC after error@1519856213) -> (continuation of Deferred@1187182342 after retry RPC after error@392970079) -> (continuation of Deferred@1272000550 after retry RPC after error@678919819) -> (continuation of Deferred@2036123204 after retry RPC after error@264018167) -> (continuation of Deferred@1294911782 after retry RPC after error@1581151534)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.109.126:45011,127.15.109.124:45285,127.15.109.125:42037) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.126:45011(127.15.109.126:45011): Connection refused: /127.15.109.126:45011,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.109.125:42037(127.15.109.125:42037): Connection refused: /127.15.109.125:42037
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.109.125:42037
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)