03:27:05.179 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:27:05.195 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQyuJ_i/test-tmp
03:27:05.198 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:27:05.203 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQyuJ_i/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:27:05.570 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:27:05.747 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:27:05.748 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:27:05.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:05.753051 25163 mini_oidc.cc:152] Starting JWKS server
03:27:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:05.759357 25163 webserver.cc:460] Webserver started at http://127.0.0.1:37887/ using document root <none> and password file <none>
03:27:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:05.759948 25163 mini_oidc.cc:189] Starting OIDC Discovery server
03:27:05.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:05.762202 25163 webserver.cc:460] Webserver started at http://127.0.0.1:45073/ using document root <none> and password file <none>
03:27:05.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T03:27:05Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:27:05.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T03:27:05Z Disabled control of system clock
03:27:05.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:05.833861 25163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/wal
03:27:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/data
03:27:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:05.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:05.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:05.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/logs
03:27:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/data/info.pb
03:27:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.254:44615
03:27:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.254
03:27:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json with env {}
03:27:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.197803 25176 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:06.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.198911 25176 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.199717 25176 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:06.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.232793 25176 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.233155 25176 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.233351 25176 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.233530 25176 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.233840 25176 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.234057 25176 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
03:27:06.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:06.272626 25176 master_runner.cc:386] Master server non-default flags:
03:27:06.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:06.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/data
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/wal
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.254:44615
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/data/info.pb
03:27:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.254
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/logs
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:06.274188 25176 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:06.276420 25176 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:06.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.300588 25182 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.301059 25183 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:07.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:06.303001 25176 server_base.cc:1034] running on GCE node
03:27:07.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:06.300943 25185 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.680004 25176 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.683537 25176 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.685134 25176 hybrid_clock.cc:648] HybridClock initialized: now 1786246027684967 us; error 95 us; skew 500 ppm
03:27:07.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.686466 25176 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:07.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.695561 25176 webserver.cc:460] Webserver started at http://127.24.146.254:45555/ using document root <none> and password file <none>
03:27:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.697057 25176 fs_manager.cc:359] Metadata directory not provided
03:27:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.697333 25176 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:07.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.697950 25176 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:07.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.704231 25176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/data/instance:
03:27:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49d4a9de9d8742a2ba7144f3cc624eba"
03:27:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:07 on dist-test-slave-77v9"
03:27:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.706632 25176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/wal/instance:
03:27:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49d4a9de9d8742a2ba7144f3cc624eba"
03:27:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:07 on dist-test-slave-77v9"
03:27:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.715816 25176 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
03:27:07.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.724458 25192 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:07.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.725991 25176 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.005s sys 0.002s
03:27:07.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.726477 25176 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/wal
03:27:07.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49d4a9de9d8742a2ba7144f3cc624eba"
03:27:07.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:07 on dist-test-slave-77v9"
03:27:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.726976 25176 fs_report.cc:383] FS layout report
03:27:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/wal
03:27:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/wal
03:27:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/data/data
03:27:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:07.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.780673 25176 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.783324 25176 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:07.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.783972 25176 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.889930 25176 rpc_server.cc:307] RPC server started. Bound to: 127.24.146.254:44615
03:27:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.890126 25248 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.146.254:44615 every 8 connection(s)
03:27:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.893610 25176 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/data/info.pb
03:27:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.901631 25163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 25176
03:27:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.901899 25249 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:27:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.902302 25163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-0/wal/instance
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.915284 25163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/wal
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/data
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/logs
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/data/info.pb
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:07.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:07.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.253:46829
03:27:07.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.253
03:27:07.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json with env {}
03:27:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.933637 25249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } has no permanent_uuid. Determining permanent_uuid...
03:27:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:07.977779 25249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } has no permanent_uuid. Determining permanent_uuid...
03:27:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:07.983314 25198 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.146.253:46829: connect: Connection refused (error 111)
03:27:07.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:07.990799 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.253:46829: Network error: Client connection negotiation failed: client connection to 127.24.146.253:46829: connect: Connection refused (error 111)
03:27:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.040540 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } attempt: 1
03:27:08.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.047031 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.253:46829: Network error: Client connection negotiation failed: client connection to 127.24.146.253:46829: connect: Connection refused (error 111)
03:27:08.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.115671 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } attempt: 2
03:27:08.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.121949 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.253:46829: Network error: Client connection negotiation failed: client connection to 127.24.146.253:46829: connect: Connection refused (error 111)
03:27:08.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.213773 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } attempt: 3
03:27:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.219256 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.253:46829: Network error: Client connection negotiation failed: client connection to 127.24.146.253:46829: connect: Connection refused (error 111)
03:27:08.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.331910 25251 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.332541 25251 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.333320 25251 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:08.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.362926 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } attempt: 4
03:27:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.368273 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.253:46829: Network error: Client connection negotiation failed: client connection to 127.24.146.253:46829: connect: Connection refused (error 111)
03:27:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.370826 25251 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.371138 25251 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.371371 25251 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.371659 25251 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.371881 25251 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.372041 25251 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
03:27:08.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.417049 25251 master_runner.cc:386] Master server non-default flags:
03:27:08.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:08.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:08.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/data
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/wal
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.253:46829
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/data/info.pb
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.253
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/logs
03:27:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.418586 25251 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.420373 25251 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:08.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.443888 25260 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:08.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.667779 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } attempt: 5
03:27:08.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.673029 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.253:46829: Network error: Client connection negotiation failed: client connection to 127.24.146.253:46829: connect: Connection refused (error 111)
03:27:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.220501 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } attempt: 6
03:27:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:09.229055 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.253:46829: Network error: Client connection negotiation failed: client connection to 127.24.146.253:46829: connect: Connection refused (error 111)
03:27:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.443928 25263 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:08.443939 25261 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:08.445034 25251 server_base.cc:1034] running on GCE node
03:27:09.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.797506 25251 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.801807 25251 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.803495 25251 hybrid_clock.cc:648] HybridClock initialized: now 1786246029803385 us; error 98 us; skew 500 ppm
03:27:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.805370 25251 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.820113 25251 webserver.cc:460] Webserver started at http://127.24.146.253:34575/ using document root <none> and password file <none>
03:27:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.821482 25251 fs_manager.cc:359] Metadata directory not provided
03:27:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.821710 25251 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.822178 25251 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.828462 25251 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/data/instance:
03:27:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9e6403d8aea4629b82272893b72f0da"
03:27:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:09 on dist-test-slave-77v9"
03:27:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.830139 25251 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/wal/instance:
03:27:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9e6403d8aea4629b82272893b72f0da"
03:27:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:09 on dist-test-slave-77v9"
03:27:09.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.839470 25251 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.001s sys 0.008s
03:27:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.846757 25274 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.848256 25251 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
03:27:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.848608 25251 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/wal
03:27:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9e6403d8aea4629b82272893b72f0da"
03:27:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:09 on dist-test-slave-77v9"
03:27:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.849047 25251 fs_report.cc:383] FS layout report
03:27:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/wal
03:27:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/wal
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/data/data
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:09.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.924538 25251 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:09.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.926210 25251 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:09.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:09.926688 25251 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:10.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.028360 25251 rpc_server.cc:307] RPC server started. Bound to: 127.24.146.253:46829
03:27:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.028478 25328 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.146.253:46829 every 8 connection(s)
03:27:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.031944 25251 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/data/info.pb
03:27:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.035622 25163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 25251
03:27:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.036345 25163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-1/wal/instance
03:27:10.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.041045 25163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:10.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:10.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/wal
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/data
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/logs
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/data/info.pb
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.252:35963
03:27:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.252
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json with env {}
03:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.043022 25330 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:27:10.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.060249 25330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } has no permanent_uuid. Determining permanent_uuid...
03:27:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.098948 25330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } has no permanent_uuid. Determining permanent_uuid...
03:27:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.123965 25330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } has no permanent_uuid. Determining permanent_uuid...
03:27:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.128333 25277 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.142513 25330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.192212 25330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 1
03:27:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.196944 25330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.265542 25330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 2
03:27:10.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.271231 25330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.290068 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } attempt: 7
03:27:10.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.307832 25249 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } has no permanent_uuid. Determining permanent_uuid...
03:27:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.315680 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.362746 25330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 3
03:27:10.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.367632 25330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.374250 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 1
03:27:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.378943 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.457162 25333 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:10.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.458042 25333 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:10.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.458809 25333 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:10.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.460414 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 2
03:27:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.465294 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.498039 25333 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.498467 25333 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:10.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.498823 25333 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:10.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.499238 25333 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:10.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.499745 25333 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:10.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.500036 25333 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
03:27:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.511195 25330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 4
03:27:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.516028 25330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.542135 25333 master_runner.cc:386] Master server non-default flags:
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/data
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/wal
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.252:35963
03:27:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
03:27:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/data/info.pb
03:27:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.252
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/logs
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.544157 25333 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.547000 25333 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.550854 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 3
03:27:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.557739 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.577193 25351 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.698522 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 4
03:27:10.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.704456 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.815542 25330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 5
03:27:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.822475 25330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:10.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.988142 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 5
03:27:10.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.993503 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.369967 25330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 6
03:27:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:11.378115 25330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:11.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.546089 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 6
03:27:11.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:11.554770 25249 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.146.252:35963: Network error: Client connection negotiation failed: client connection to 127.24.146.252:35963: connect: Connection refused (error 111)
03:27:11.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.577723 25348 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:10.579540 25333 server_base.cc:1034] running on GCE node
03:27:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:10.583384 25347 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.945199 25333 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.948659 25333 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.950162 25333 hybrid_clock.cc:648] HybridClock initialized: now 1786246031950115 us; error 67 us; skew 500 ppm
03:27:11.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.951314 25333 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:11.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.959537 25333 webserver.cc:460] Webserver started at http://127.24.146.252:35547/ using document root <none> and password file <none>
03:27:11.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.960532 25333 fs_manager.cc:359] Metadata directory not provided
03:27:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.960738 25333 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.961308 25333 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.966495 25333 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/data/instance:
03:27:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0422a0d9371941678d6a592e836cac0d"
03:27:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:11 on dist-test-slave-77v9"
03:27:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.967650 25333 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/wal/instance:
03:27:11.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0422a0d9371941678d6a592e836cac0d"
03:27:11.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:11 on dist-test-slave-77v9"
03:27:11.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.976089 25333 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
03:27:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.983376 25362 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.984736 25333 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
03:27:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.985183 25333 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/wal
03:27:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0422a0d9371941678d6a592e836cac0d"
03:27:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:11 on dist-test-slave-77v9"
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:11.985538 25333 fs_report.cc:383] FS layout report
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/wal
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/wal
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/data/data
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.034160 25333 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.036041 25333 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.036641 25333 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:12.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.132102 25333 rpc_server.cc:307] RPC server started. Bound to: 127.24.146.252:35963
03:27:12.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.132369 25417 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.146.252:35963 every 8 connection(s)
03:27:12.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.135993 25333 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/data/info.pb
03:27:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.143975 25419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:27:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.146603 25163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 25333
03:27:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.147117 25163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/master-2/wal/instance
03:27:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.159672 25419 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } has no permanent_uuid. Determining permanent_uuid...
03:27:12.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.194432 25419 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } has no permanent_uuid. Determining permanent_uuid...
03:27:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.212792 25419 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } has no permanent_uuid. Determining permanent_uuid...
03:27:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.237663 25419 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d: Bootstrap starting.
03:27:12.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.245584 25419 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d: Neither blocks nor log segments found. Creating new log.
03:27:12.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.247946 25419 log.cc:824] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d: Log is configured to *not* fsync() on all Append() calls
03:27:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.254850 25419 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d: No bootstrap required, opened a new log
03:27:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.277401 25419 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.278374 25419 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:27:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.278669 25419 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0422a0d9371941678d6a592e836cac0d, State: Initialized, Role: FOLLOWER
03:27:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.279588 25419 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.282796 25431 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.283507 25431 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: This master's current role is: FOLLOWER
03:27:12.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.284723 25419 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: configured and running, proceeding with master startup.
03:27:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:12.293988 25369 tablet.cc:2369] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:27:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:12.317595 25442 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:12.318418 25442 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:12.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.311328 25333 master_runner.cc:186] Error getting master registration for 127.24.146.254:44615: OK, instance_id { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" instance_seqno: 1786246027862423 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:27:12.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.350979 25333 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:27:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.438819 25330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 7
03:27:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.461812 25330 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: Bootstrap starting.
03:27:12.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.467842 25330 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: Neither blocks nor log segments found. Creating new log.
03:27:12.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.470196 25330 log.cc:824] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: Log is configured to *not* fsync() on all Append() calls
03:27:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.476704 25330 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: No bootstrap required, opened a new log
03:27:12.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.504264 25330 raft_consensus.cc:348] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.505180 25330 raft_consensus.cc:374] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:27:12.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.505482 25330 raft_consensus.cc:729] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9e6403d8aea4629b82272893b72f0da, State: Initialized, Role: FOLLOWER
03:27:12.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.506546 25330 consensus_queue.cc:260] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.511264 25446 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.513644 25446 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [sys.catalog]: This master's current role is: FOLLOWER
03:27:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.515118 25330 sys_catalog.cc:564] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [sys.catalog]: configured and running, proceeding with master startup.
03:27:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:12.553682 25457 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:12.554693 25457 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:12.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.547484 25251 master_runner.cc:186] Error getting master registration for 127.24.146.254:44615: OK, instance_id { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" instance_seqno: 1786246027862423 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:27:12.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.586964 25251 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:27:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.588442 25249 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } attempt: 7
03:27:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.620353 25249 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba: Bootstrap starting.
03:27:12.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.630101 25249 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba: Neither blocks nor log segments found. Creating new log.
03:27:12.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.633697 25249 log.cc:824] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba: Log is configured to *not* fsync() on all Append() calls
03:27:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.643095 25249 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba: No bootstrap required, opened a new log
03:27:12.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.682669 25249 raft_consensus.cc:348] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.683631 25249 raft_consensus.cc:374] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:27:12.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.683935 25249 raft_consensus.cc:729] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 49d4a9de9d8742a2ba7144f3cc624eba, State: Initialized, Role: FOLLOWER
03:27:12.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.684931 25249 consensus_queue.cc:260] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:12.687625 25282 tablet.cc:2369] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:27:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.690263 25461 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.691469 25461 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [sys.catalog]: This master's current role is: FOLLOWER
03:27:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.692677 25249 sys_catalog.cc:564] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [sys.catalog]: configured and running, proceeding with master startup.
03:27:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.715862 25431 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:27:12.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.716504 25431 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.731269 25221 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0422a0d9371941678d6a592e836cac0d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" is_pre_election: true
03:27:12.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.732383 25221 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0422a0d9371941678d6a592e836cac0d in term 0.
03:27:12.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.734331 25365 leader_election.cc:302] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0422a0d9371941678d6a592e836cac0d, 49d4a9de9d8742a2ba7144f3cc624eba; no voters:
03:27:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.740450 25303 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0422a0d9371941678d6a592e836cac0d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9e6403d8aea4629b82272893b72f0da" is_pre_election: true
03:27:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.742669 25303 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0422a0d9371941678d6a592e836cac0d in term 0.
03:27:12.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.756742 25431 leader_election.cc:288] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 49d4a9de9d8742a2ba7144f3cc624eba (127.24.146.254:44615), b9e6403d8aea4629b82272893b72f0da (127.24.146.253:46829)
03:27:12.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.757414 25431 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 0 FOLLOWER]: Leader pre-election won for term 1
03:27:12.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.757766 25431 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:27:12.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.758085 25431 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 0 FOLLOWER]: Advancing to term 1
03:27:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.763890 25475 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:27:12.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.771718 25431 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.774982 25221 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0422a0d9371941678d6a592e836cac0d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "49d4a9de9d8742a2ba7144f3cc624eba"
03:27:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.775583 25221 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [term 0 FOLLOWER]: Advancing to term 1
03:27:12.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.793633 25221 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0422a0d9371941678d6a592e836cac0d in term 1.
03:27:12.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.795032 25431 leader_election.cc:288] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [CANDIDATE]: Term 1 election: Requested vote from peers 49d4a9de9d8742a2ba7144f3cc624eba (127.24.146.254:44615), b9e6403d8aea4629b82272893b72f0da (127.24.146.253:46829)
03:27:12.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.797479 25303 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0422a0d9371941678d6a592e836cac0d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9e6403d8aea4629b82272893b72f0da"
03:27:12.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.798128 25303 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [term 0 FOLLOWER]: Advancing to term 1
03:27:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:12.800642 25200 tablet.cc:2369] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:27:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.805554 25365 leader_election.cc:302] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0422a0d9371941678d6a592e836cac0d, 49d4a9de9d8742a2ba7144f3cc624eba; no voters:
03:27:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.807894 25176 master_runner.cc:232] Existing masters have differing terms: 0 vs 1
03:27:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.808585 25176 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:27:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.810873 25431 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 1 FOLLOWER]: Leader election won for term 1
03:27:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.813051 25303 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0422a0d9371941678d6a592e836cac0d in term 1.
03:27:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.815641 25431 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [term 1 LEADER]: Becoming Leader. State: Replica: 0422a0d9371941678d6a592e836cac0d, State: Running, Role: LEADER
03:27:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.817655 25431 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } }
03:27:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.832262 25472 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0422a0d9371941678d6a592e836cac0d. Latest consensus state: current_term: 1 leader_uuid: "0422a0d9371941678d6a592e836cac0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.833355 25472 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: This master's current role is: LEADER
03:27:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.836354 25477 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:27:12.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.843953 25477 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:27:12.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.873880 25303 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [term 1 FOLLOWER]: Refusing update from remote peer 0422a0d9371941678d6a592e836cac0d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:27:12.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.874461 25221 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [term 1 FOLLOWER]: Refusing update from remote peer 0422a0d9371941678d6a592e836cac0d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:27:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.876590 25431 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:27:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.877610 25472 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [LEADER]: Connected to new peer: Peer: permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:27:12.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.918560 25461 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0422a0d9371941678d6a592e836cac0d. Latest consensus state: current_term: 1 leader_uuid: "0422a0d9371941678d6a592e836cac0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.919570 25461 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [sys.catalog]: This master's current role is: FOLLOWER
03:27:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.926911 25478 mvcc.cc:205] Tried to move back new op lower bound from 7316463750619369472 to 7316463750438109184. Current Snapshot: MvccSnapshot[applied={T|T < 7316463750619369472}]
03:27:12.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.942982 25461 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0422a0d9371941678d6a592e836cac0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.943857 25461 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba [sys.catalog]: This master's current role is: FOLLOWER
03:27:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.956864 25472 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0422a0d9371941678d6a592e836cac0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.958019 25472 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: This master's current role is: LEADER
03:27:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.959023 25477 catalog_manager.cc:1347] Generated new cluster ID: 4d6f9c292e7c47ed854a1ffc7211fc8c
03:27:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.959301 25477 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:27:12.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.965903 25446 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0422a0d9371941678d6a592e836cac0d. Latest consensus state: current_term: 1 leader_uuid: "0422a0d9371941678d6a592e836cac0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.967325 25446 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [sys.catalog]: This master's current role is: FOLLOWER
03:27:12.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.973325 25431 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0422a0d9371941678d6a592e836cac0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.976871 25431 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d [sys.catalog]: This master's current role is: LEADER
03:27:12.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.987627 25446 sys_catalog.cc:455] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0422a0d9371941678d6a592e836cac0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "49d4a9de9d8742a2ba7144f3cc624eba" member_type: VOTER last_known_addr { host: "127.24.146.254" port: 44615 } } peers { permanent_uuid: "b9e6403d8aea4629b82272893b72f0da" member_type: VOTER last_known_addr { host: "127.24.146.253" port: 46829 } } peers { permanent_uuid: "0422a0d9371941678d6a592e836cac0d" member_type: VOTER last_known_addr { host: "127.24.146.252" port: 35963 } } }
03:27:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:12.988760 25446 sys_catalog.cc:458] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da [sys.catalog]: This master's current role is: FOLLOWER
03:27:13.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.019169 25477 catalog_manager.cc:1370] Generated new certificate authority record
03:27:13.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.022769 25477 catalog_manager.cc:1504] Loading token signing keys...
03:27:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.067088 25477 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 0422a0d9371941678d6a592e836cac0d: Generated new TSK 0
03:27:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.068622 25477 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:27:13.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.101470 25163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/wal
03:27:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/data
03:27:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/logs
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/data/info.pb
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.193:0
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.146.193
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.193
03:27:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:27:13.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.560956 25457 catalog_manager.cc:1259] Loaded cluster ID: 4d6f9c292e7c47ed854a1ffc7211fc8c
03:27:13.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.562172 25457 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: loading cluster ID for follower catalog manager: success
03:27:13.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.570639 25457 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: acquiring CA information for follower catalog manager: success
03:27:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.577052 25457 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b9e6403d8aea4629b82272893b72f0da: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:27:13.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:13.653306 25489 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:13.654045 25489 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:13.654940 25489 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:13.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:13.721195 25489 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:13.722961 25489 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.146.193
03:27:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.790834 25475 catalog_manager.cc:1259] Loaded cluster ID: 4d6f9c292e7c47ed854a1ffc7211fc8c
03:27:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.791513 25475 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba: loading cluster ID for follower catalog manager: success
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.794955 25489 tablet_server_runner.cc:78] Tablet server non-default flags:
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/data
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/wal
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.193:0
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/data/info.pb
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.193
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.146.193
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/logs
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.796636 25489 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:13.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.799043 25489 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:13.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.815002 25475 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba: acquiring CA information for follower catalog manager: success
03:27:13.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.829118 25475 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 49d4a9de9d8742a2ba7144f3cc624eba: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:27:13.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:13.850046 25498 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:15.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:13.855873 25489 server_base.cc:1034] running on GCE node
03:27:15.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:13.866088 25497 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:15.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:13.853585 25502 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:15.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.554764 25489 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:15.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.557915 25489 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:15.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.560329 25489 hybrid_clock.cc:648] HybridClock initialized: now 1786246035560265 us; error 86 us; skew 500 ppm
03:27:15.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.561650 25489 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.575155 25489 webserver.cc:460] Webserver started at http://127.24.146.193:40421/ using document root <none> and password file <none>
03:27:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.577080 25489 fs_manager.cc:359] Metadata directory not provided
03:27:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.577464 25489 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:15.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.578512 25489 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.590715 25489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/data/instance:
03:27:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dece40493dc04ad18f4c79422203c08d"
03:27:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:15 on dist-test-slave-77v9"
03:27:15.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.592465 25489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/wal/instance:
03:27:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dece40493dc04ad18f4c79422203c08d"
03:27:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:15 on dist-test-slave-77v9"
03:27:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.603412 25489 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.003s
03:27:15.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.612964 25508 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:15.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.615070 25489 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.006s sys 0.001s
03:27:15.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.615550 25489 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/wal
03:27:15.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dece40493dc04ad18f4c79422203c08d"
03:27:15.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:15 on dist-test-slave-77v9"
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.616255 25489 fs_report.cc:383] FS layout report
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/wal
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/wal
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/data/data
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.678330 25489 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.680596 25489 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.681620 25489 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.689685 25489 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:27:15.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.697979 25489 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:27:15.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.698339 25489 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:27:15.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.698690 25489 ts_tablet_manager.cc:610] Registered 0 tablets
03:27:15.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.698926 25489 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:27:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.966725 25489 rpc_server.cc:307] RPC server started. Bound to: 127.24.146.193:45379
03:27:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.966856 25626 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.146.193:45379 every 8 connection(s)
03:27:15.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.971231 25489 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/data/info.pb
03:27:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.983341 25163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 25489
03:27:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:15.984081 25163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-0/wal/instance
03:27:16.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.018473 25163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/wal
03:27:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/data
03:27:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/logs
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/data/info.pb
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.194:0
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.146.194
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.194
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:27:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.125454 25627 heartbeater.cc:347] Connected to a master server at 127.24.146.252:35963
03:27:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.126160 25627 heartbeater.cc:464] Registering TS with master...
03:27:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.129134 25627 heartbeater.cc:511] Master 127.24.146.252:35963 requested a full tablet report, sending...
03:27:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.133500 25382 ts_manager.cc:194] Registered new tserver with Master: dece40493dc04ad18f4c79422203c08d (127.24.146.193:45379)
03:27:16.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.137648 25382 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.146.193:34047
03:27:16.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.156054 25628 heartbeater.cc:347] Connected to a master server at 127.24.146.253:46829
03:27:16.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.156499 25628 heartbeater.cc:464] Registering TS with master...
03:27:16.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.158123 25628 heartbeater.cc:511] Master 127.24.146.253:46829 requested a full tablet report, sending...
03:27:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.163264 25293 ts_manager.cc:194] Registered new tserver with Master: dece40493dc04ad18f4c79422203c08d (127.24.146.193:45379)
03:27:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.179162 25633 heartbeater.cc:347] Connected to a master server at 127.24.146.254:44615
03:27:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.179698 25633 heartbeater.cc:464] Registering TS with master...
03:27:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.180559 25633 heartbeater.cc:511] Master 127.24.146.254:44615 requested a full tablet report, sending...
03:27:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.185132 25211 ts_manager.cc:194] Registered new tserver with Master: dece40493dc04ad18f4c79422203c08d (127.24.146.193:45379)
03:27:16.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:16.627619 25634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:16.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:16.628239 25634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:16.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:16.628942 25634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:16.669000 25634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:16.670104 25634 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.146.194
03:27:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.709949 25634 tablet_server_runner.cc:78] Tablet server non-default flags:
03:27:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/data
03:27:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/wal
03:27:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.194:0
03:27:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/data/info.pb
03:27:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.194
03:27:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.146.194
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/logs
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:16.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:16.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.711889 25634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:16.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.714030 25634 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:16.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:16.742934 25649 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:17.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:17.146031 25627 heartbeater.cc:503] Master 127.24.146.252:35963 was elected leader, sending a full tablet report...
03:27:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:16.743844 25647 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:16.746548 25646 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:16.745265 25634 server_base.cc:1034] running on GCE node
03:27:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.181989 25634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:18.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.185045 25634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:18.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.186611 25634 hybrid_clock.cc:648] HybridClock initialized: now 1786246038186575 us; error 47 us; skew 500 ppm
03:27:18.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.187651 25634 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:18.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.198500 25634 webserver.cc:460] Webserver started at http://127.24.146.194:44229/ using document root <none> and password file <none>
03:27:18.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.199798 25634 fs_manager.cc:359] Metadata directory not provided
03:27:18.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.200086 25634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:18.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.201280 25634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.207813 25634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/data/instance:
03:27:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "00577e282aae424d83f63bd7141bbc89"
03:27:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:18 on dist-test-slave-77v9"
03:27:18.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.209439 25634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/wal/instance:
03:27:18.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "00577e282aae424d83f63bd7141bbc89"
03:27:18.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:18 on dist-test-slave-77v9"
03:27:18.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.221522 25634 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.010s sys 0.001s
03:27:18.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.230268 25658 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:18.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.231907 25634 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s
03:27:18.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.232776 25634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/wal
03:27:18.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "00577e282aae424d83f63bd7141bbc89"
03:27:18.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:18 on dist-test-slave-77v9"
03:27:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.233363 25634 fs_report.cc:383] FS layout report
03:27:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/wal
03:27:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/wal
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/data/data
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:18.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.282509 25634 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.284633 25634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:18.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.285342 25634 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.289312 25634 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:27:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.295751 25634 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:27:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.296051 25634 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:27:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.296428 25634 ts_tablet_manager.cc:610] Registered 0 tablets
03:27:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.296634 25634 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:27:18.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.500032 25634 rpc_server.cc:307] RPC server started. Bound to: 127.24.146.194:36275
03:27:18.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.500180 25779 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.146.194:36275 every 8 connection(s)
03:27:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.503028 25634 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/data/info.pb
03:27:18.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.504767 25163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 25634
03:27:18.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.505455 25163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-1/wal/instance
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.525532 25163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/wal
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/data
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/logs
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/data/info.pb
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.195:0
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.146.195
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.195
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:27:18.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.588359 25781 heartbeater.cc:347] Connected to a master server at 127.24.146.253:46829
03:27:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.588515 25780 heartbeater.cc:347] Connected to a master server at 127.24.146.252:35963
03:27:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.589303 25780 heartbeater.cc:464] Registering TS with master...
03:27:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.589751 25781 heartbeater.cc:464] Registering TS with master...
03:27:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.590798 25780 heartbeater.cc:511] Master 127.24.146.252:35963 requested a full tablet report, sending...
03:27:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.591146 25781 heartbeater.cc:511] Master 127.24.146.253:46829 requested a full tablet report, sending...
03:27:18.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.592921 25784 heartbeater.cc:347] Connected to a master server at 127.24.146.254:44615
03:27:18.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.593267 25784 heartbeater.cc:464] Registering TS with master...
03:27:18.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.594187 25784 heartbeater.cc:511] Master 127.24.146.254:44615 requested a full tablet report, sending...
03:27:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.594691 25293 ts_manager.cc:194] Registered new tserver with Master: 00577e282aae424d83f63bd7141bbc89 (127.24.146.194:36275)
03:27:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.596540 25382 ts_manager.cc:194] Registered new tserver with Master: 00577e282aae424d83f63bd7141bbc89 (127.24.146.194:36275)
03:27:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.597775 25211 ts_manager.cc:194] Registered new tserver with Master: 00577e282aae424d83f63bd7141bbc89 (127.24.146.194:36275)
03:27:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:18.598263 25382 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.146.194:33277
03:27:18.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:18.973011 25792 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:18.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:18.973763 25792 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:18.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:18.974557 25792 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:19.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:19.012336 25792 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:19.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:19.013332 25792 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.146.195
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:19.066879 25792 tablet_server_runner.cc:78] Tablet server non-default flags:
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.146.212:40951
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/data
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/wal
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.146.195:0
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/data/info.pb
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.146.195
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.146.254:44615,127.24.146.253:46829,127.24.146.252:35963
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.146.195
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/logs
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:19.068161 25792 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:19.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:19.069875 25792 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:19.098510 25802 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:19.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:19.604265 25780 heartbeater.cc:503] Master 127.24.146.252:35963 was elected leader, sending a full tablet report...
03:27:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:19.099186 25800 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:19.099270 25799 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:19.102460 25792 server_base.cc:1034] running on GCE node
03:27:20.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.475593 25792 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.479117 25792 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.480710 25792 hybrid_clock.cc:648] HybridClock initialized: now 1786246040480674 us; error 61 us; skew 500 ppm
03:27:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.482167 25792 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:20.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.490988 25792 webserver.cc:460] Webserver started at http://127.24.146.195:42723/ using document root <none> and password file <none>
03:27:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.492324 25792 fs_manager.cc:359] Metadata directory not provided
03:27:20.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.492621 25792 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:20.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.493397 25792 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.502460 25792 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/data/instance:
03:27:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd2b57e04eba42e09174f43d710b6c0e"
03:27:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:20 on dist-test-slave-77v9"
03:27:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.504351 25792 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/wal/instance:
03:27:20.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd2b57e04eba42e09174f43d710b6c0e"
03:27:20.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:20 on dist-test-slave-77v9"
03:27:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.516498 25792 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.006s sys 0.004s
03:27:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.524042 25810 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.525643 25792 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.007s sys 0.000s
03:27:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.526051 25792 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/wal
03:27:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd2b57e04eba42e09174f43d710b6c0e"
03:27:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:20 on dist-test-slave-77v9"
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.526474 25792 fs_report.cc:383] FS layout report
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/wal
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/wal
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/data/data
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:20.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.589013 25792 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:20.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.590728 25792 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:20.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.591279 25792 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:20.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.594991 25792 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:27:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.600050 25792 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:27:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.600330 25792 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:27:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.600607 25792 ts_tablet_manager.cc:610] Registered 0 tablets
03:27:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.600760 25792 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:27:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.863597 25792 rpc_server.cc:307] RPC server started. Bound to: 127.24.146.195:37769
03:27:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.863619 25930 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.146.195:37769 every 8 connection(s)
03:27:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.866555 25792 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/data/info.pb
03:27:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.872833 25163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 25792
03:27:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.873380 25163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster5728790438284241484/ts-2/wal/instance
03:27:20.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.971421 25932 heartbeater.cc:347] Connected to a master server at 127.24.146.253:46829
03:27:20.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.971992 25932 heartbeater.cc:464] Registering TS with master...
03:27:20.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.972520 25936 heartbeater.cc:347] Connected to a master server at 127.24.146.254:44615
03:27:20.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.972920 25936 heartbeater.cc:464] Registering TS with master...
03:27:20.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.973433 25932 heartbeater.cc:511] Master 127.24.146.253:46829 requested a full tablet report, sending...
03:27:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.973486 25931 heartbeater.cc:347] Connected to a master server at 127.24.146.252:35963
03:27:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.973985 25931 heartbeater.cc:464] Registering TS with master...
03:27:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.973999 25936 heartbeater.cc:511] Master 127.24.146.254:44615 requested a full tablet report, sending...
03:27:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.974906 25931 heartbeater.cc:511] Master 127.24.146.252:35963 requested a full tablet report, sending...
03:27:20.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.978448 25293 ts_manager.cc:194] Registered new tserver with Master: cd2b57e04eba42e09174f43d710b6c0e (127.24.146.195:37769)
03:27:20.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.978806 25211 ts_manager.cc:194] Registered new tserver with Master: cd2b57e04eba42e09174f43d710b6c0e (127.24.146.195:37769)
03:27:20.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.978627 25382 ts_manager.cc:194] Registered new tserver with Master: cd2b57e04eba42e09174f43d710b6c0e (127.24.146.195:37769)
03:27:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:20.983932 25382 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.146.195:40837
03:27:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:21.001149 25163 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
03:27:21.017 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:27:21.018 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:27:21.033 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.146.254"
port: 44615
}
bound_http_address {
host: "127.24.146.254"
port: 45555
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.146.253"
port: 46829
}
bound_http_address {
host: "127.24.146.253"
port: 34575
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.146.252"
port: 35963
}
bound_http_address {
host: "127.24.146.252"
port: 35547
}
}
}
03:27:21.041 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:27:21.045 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.24.146.193"
port: 45379
}
bound_http_address {
host: "127.24.146.193"
port: 40421
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.24.146.194"
port: 36275
}
bound_http_address {
host: "127.24.146.194"
port: 44229
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.24.146.195"
port: 37769
}
bound_http_address {
host: "127.24.146.195"
port: 42723
}
}
}
03:27:21.045 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:27:21.275 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:27:21.437 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQyuJ_i/test-tmp
03:27:21.439 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:27:21.439 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQyuJ_i/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:27:21.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:27:21.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:21.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:27:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:21.918286 25944 mini_oidc.cc:152] Starting JWKS server
03:27:21.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:21.922693 25944 webserver.cc:460] Webserver started at http://127.0.0.1:40777/ using document root <none> and password file <none>
03:27:21.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:21.923169 25944 mini_oidc.cc:189] Starting OIDC Discovery server
03:27:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:21.926206 25944 webserver.cc:460] Webserver started at http://127.0.0.1:44529/ using document root <none> and password file <none>
03:27:21.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:21.992483 25931 heartbeater.cc:503] Master 127.24.146.252:35963 was elected leader, sending a full tablet report...
03:27:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:27:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/principal' for realm 'KRBTEST.COM',
03:27:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:27:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:22 dist-test-slave-77v9 krb5kdc[25957](info): setting up network...
03:27:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:27:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:22 dist-test-slave-77v9 krb5kdc[25957](info): set up 2 sockets
03:27:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:22 dist-test-slave-77v9 krb5kdc[25957](info): commencing operation
03:27:22.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:27:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.234464 25944 mini_kdc.cc:121] Time spent starting KDC: real 1.277s user 0.004s sys 0.034s
03:27:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:27:23.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:27:23.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:27:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:27:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:27:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:27:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:27:23.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:27:23.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:27:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:27:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/test-user.keytab.
03:27:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/test-user.keytab.
03:27:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:23 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246043, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:27:23.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T03:27:23Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:27:23.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T03:27:23Z Disabled control of system clock
03:27:23.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.86.62@KRBTEST.COM; defaulting to no policy
03:27:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:27:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.86.62@KRBTEST.COM" created.
03:27:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:27:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.86.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.62.keytab.
03:27:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.86.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.62.keytab.
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:23.508538 25944 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/logs
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/info.pb
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:23.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.62:36625
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.62
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.62.keytab
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.62
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:27:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.883541 25970 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.884270 25970 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.885118 25970 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.925902 25970 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:27:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.926246 25970 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.926781 25970 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.927160 25970 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.927501 25970 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.927886 25970 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:23.928153 25970 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40777/jwks.json
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:23.973769 25970 master_runner.cc:386] Master server non-default flags:
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.62.keytab
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.62
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.62:36625
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/info.pb
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.62
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/logs
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:23.975523 25970 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:23.977080 25970 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:24.003664 25977 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:24.003863 25980 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:24.004889 25978 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:25.433221 25979 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:27:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.435353 25970 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:27:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.438651 25970 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.441911 25970 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.443738 25970 hybrid_clock.cc:648] HybridClock initialized: now 1786246045443599 us; error 127 us; skew 500 ppm
03:27:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.444984 25970 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:25.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:25 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246045, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.454258 25970 init.cc:377] Logged in from keytab as kudu/127.25.86.62@KRBTEST.COM (short username kudu)
03:27:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.462764 25970 webserver.cc:460] Webserver started at http://127.25.86.62:39799/ using document root <none> and password file <none>
03:27:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.463898 25970 fs_manager.cc:359] Metadata directory not provided
03:27:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.464105 25970 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.464533 25970 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:25.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.472126 25970 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/instance:
03:27:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1499c39a1a345968561ec494147b159"
03:27:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:25 on dist-test-slave-77v9"
03:27:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.473920 25970 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal/instance:
03:27:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1499c39a1a345968561ec494147b159"
03:27:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:25 on dist-test-slave-77v9"
03:27:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.484180 25970 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.005s sys 0.005s
03:27:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.492158 25988 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:25.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.494132 25970 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.003s sys 0.004s
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.494616 25970 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1499c39a1a345968561ec494147b159"
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:25 on dist-test-slave-77v9"
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.495095 25970 fs_report.cc:383] FS layout report
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/data
03:27:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:25.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.545574 25970 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.553962 25970 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.554510 25970 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.677982 25970 rpc_server.cc:307] RPC server started. Bound to: 127.25.86.62:36625
03:27:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.678062 26041 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.86.62:36625 every 8 connection(s)
03:27:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.681048 25970 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/info.pb
03:27:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.687924 26042 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:27:25.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.689240 25944 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 25970
03:27:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.689805 25944 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal/instance
03:27:25.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.702627 26042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } has no permanent_uuid. Determining permanent_uuid...
03:27:25.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.86.61@KRBTEST.COM; defaulting to no policy
03:27:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:27:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.86.61@KRBTEST.COM" created.
03:27:25.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:25 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246045, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.62@KRBTEST.COM for kudu/127.25.86.62@KRBTEST.COM
03:27:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:27:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.86.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.61.keytab.
03:27:25.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.86.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.61.keytab.
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.763684 25944 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/logs
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/info.pb
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.61:42583
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.61
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.61.keytab
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.61
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:27:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.771116 26042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } has no permanent_uuid. Determining permanent_uuid...
03:27:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:25.774722 25990 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:25.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:25.780867 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.61:42583: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.830629 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } attempt: 1
03:27:25.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:25.835625 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.61:42583: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:25.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:25.904183 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } attempt: 2
03:27:25.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:25.909998 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.61:42583: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:26.001741 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } attempt: 3
03:27:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.008955 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.61:42583: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:26.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:26.152581 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } attempt: 4
03:27:26.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.157805 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.61:42583: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.208066 26050 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.208777 26050 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.209517 26050 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:26.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.251685 26050 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:27:26.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.252014 26050 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.252720 26050 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.253198 26050 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.253813 26050 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.254276 26050 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.254573 26050 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40777/jwks.json
03:27:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:26.300199 26050 master_runner.cc:386] Master server non-default flags:
03:27:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.61.keytab
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.61
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.61:42583
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/info.pb
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.61
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/logs
03:27:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:26.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:26.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:26.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:26.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:26.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:26.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:26.301918 26050 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:26.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:26.303437 26050 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:26.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.323463 26061 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:26.457625 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } attempt: 5
03:27:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.462451 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.61:42583: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.010126 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } attempt: 6
03:27:27.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:27.020635 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.61:42583: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.323545 26059 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:26.325631 26050 server_base.cc:1034] running on GCE node
03:27:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:26.323652 26058 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:27.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.745819 26050 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:27.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.750033 26050 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:27.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.751680 26050 hybrid_clock.cc:648] HybridClock initialized: now 1786246047751632 us; error 67 us; skew 500 ppm
03:27:27.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.753122 26050 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:27 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246047, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:27.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.761253 26050 init.cc:377] Logged in from keytab as kudu/127.25.86.61@KRBTEST.COM (short username kudu)
03:27:27.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.771039 26050 webserver.cc:460] Webserver started at http://127.25.86.61:35315/ using document root <none> and password file <none>
03:27:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.772184 26050 fs_manager.cc:359] Metadata directory not provided
03:27:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.772414 26050 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:27.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.773172 26050 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.779983 26050 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/instance:
03:27:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c804196449a449919c8f1b9437fde2b1"
03:27:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:27 on dist-test-slave-77v9"
03:27:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.781359 26050 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal/instance:
03:27:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c804196449a449919c8f1b9437fde2b1"
03:27:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:27 on dist-test-slave-77v9"
03:27:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.794765 26050 fs_manager.cc:691] Time spent creating directory manager: real 0.012s user 0.002s sys 0.008s
03:27:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.804633 26074 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.806130 26050 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.002s sys 0.005s
03:27:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.806484 26050 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c804196449a449919c8f1b9437fde2b1"
03:27:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:27 on dist-test-slave-77v9"
03:27:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.806814 26050 fs_report.cc:383] FS layout report
03:27:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/data
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.886298 26050 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:27.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.897818 26050 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:27.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:27.898646 26050 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.022778 26050 rpc_server.cc:307] RPC server started. Bound to: 127.25.86.61:42583
03:27:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.023450 26130 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.86.61:42583 every 8 connection(s)
03:27:28.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.026340 26050 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/info.pb
03:27:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.030994 25944 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 26050
03:27:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.031494 25944 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal/instance
03:27:28.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.034947 26131 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:27:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.061436 26131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } has no permanent_uuid. Determining permanent_uuid...
03:27:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.86.60@KRBTEST.COM; defaulting to no policy
03:27:28.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.081668 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } attempt: 7
03:27:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:27:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.86.60@KRBTEST.COM" created.
03:27:28.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:28 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246047, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.61@KRBTEST.COM for kudu/127.25.86.62@KRBTEST.COM
03:27:28.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:28 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246045, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.62@KRBTEST.COM for kudu/127.25.86.61@KRBTEST.COM
03:27:28.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.125433 26131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } has no permanent_uuid. Determining permanent_uuid...
03:27:28.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:27:28.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.86.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.60.keytab.
03:27:28.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.86.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.60.keytab.
03:27:28.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.137058 26042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } has no permanent_uuid. Determining permanent_uuid...
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.138516 25944 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.25.86.60: real 0.104s user 0.005s sys 0.037s
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.141229 25944 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/logs
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/info.pb
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.60:37417
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.60
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.60.keytab
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.60
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:27:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:28 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246047, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.61@KRBTEST.COM for kudu/127.25.86.61@KRBTEST.COM
03:27:28.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.167553 26131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } has no permanent_uuid. Determining permanent_uuid...
03:27:28.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.167943 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.171162 26076 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.178699 26131 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.226552 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 1
03:27:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.228472 26131 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 1
03:27:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.231638 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.233000 26131 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.301410 26131 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 2
03:27:28.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.307005 26131 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.313231 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 2
03:27:28.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.325232 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.398542 26131 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 3
03:27:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.405146 26131 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.410945 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 3
03:27:28.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.418169 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.548774 26131 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 4
03:27:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.554997 26131 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.558907 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 4
03:27:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.564317 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.696624 26142 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.697685 26142 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.698431 26142 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.735764 26142 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:27:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.736084 26142 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.736616 26142 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:28.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.737047 26142 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:28.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.737540 26142 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.737916 26142 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.738191 26142 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40777/jwks.json
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.780694 26142 master_runner.cc:386] Master server non-default flags:
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.60.keytab
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.60
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.60:37417
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/info.pb
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.60
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/logs
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.782420 26142 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.783649 26142 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.836100 26156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.848122 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 5
03:27:28.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:28.854449 26131 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 5
03:27:28.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.855235 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:28.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.861284 26131 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:29.407896 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 6
03:27:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:29.408861 26131 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 6
03:27:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:29.435406 26131 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:29.442209 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.475680 26042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 7
03:27:30.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.496053 26131 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } attempt: 7
03:27:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:30.505065 26042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:30.532306 26131 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.86.60:37417: Network error: Client connection negotiation failed: client connection to 127.25.86.60:37417: connect: Connection refused (error 111)
03:27:30.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.836160 26152 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:30.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:28.837801 26154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:30.601858 26155 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1764 milliseconds
03:27:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.602041 26142 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:27:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.603457 26142 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.607532 26142 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:30.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.611268 26142 hybrid_clock.cc:648] HybridClock initialized: now 1786246050611216 us; error 56 us; skew 500 ppm
03:27:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.612407 26142 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:30 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246050, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.629084 26142 init.cc:377] Logged in from keytab as kudu/127.25.86.60@KRBTEST.COM (short username kudu)
03:27:30.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.649144 26142 webserver.cc:460] Webserver started at http://127.25.86.60:33367/ using document root <none> and password file <none>
03:27:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.650547 26142 fs_manager.cc:359] Metadata directory not provided
03:27:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.650811 26142 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.651422 26142 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:27:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.658659 26142 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/instance:
03:27:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82440bfc54d4421c9812da6bb7062fb9"
03:27:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:30 on dist-test-slave-77v9"
03:27:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.661113 26142 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal/instance:
03:27:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82440bfc54d4421c9812da6bb7062fb9"
03:27:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:30 on dist-test-slave-77v9"
03:27:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.683928 26142 fs_manager.cc:691] Time spent creating directory manager: real 0.022s user 0.006s sys 0.012s
03:27:30.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.697602 26176 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:30.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.699242 26142 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.008s sys 0.003s
03:27:30.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.699764 26142 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:30.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82440bfc54d4421c9812da6bb7062fb9"
03:27:30.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:30 on dist-test-slave-77v9"
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.700578 26142 fs_report.cc:383] FS layout report
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/data
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:30.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.793754 26142 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.803733 26142 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:30.804728 26142 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:31.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.260360 26142 rpc_server.cc:307] RPC server started. Bound to: 127.25.86.60:37417
03:27:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.260950 26229 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.86.60:37417 every 8 connection(s)
03:27:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.267815 26142 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/info.pb
03:27:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.274379 25944 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 26142
03:27:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.275339 25944 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal/instance
03:27:31.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.286604 26230 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:27:31.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.332279 26230 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } has no permanent_uuid. Determining permanent_uuid...
03:27:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:31 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246043, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.86.60@KRBTEST.COM
03:27:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:31 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246050, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.60@KRBTEST.COM for kudu/127.25.86.62@KRBTEST.COM
03:27:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.410480 26230 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } has no permanent_uuid. Determining permanent_uuid...
03:27:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:31 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246043, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.86.61@KRBTEST.COM
03:27:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:31 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246050, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.60@KRBTEST.COM for kudu/127.25.86.61@KRBTEST.COM
03:27:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.486477 26230 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } has no permanent_uuid. Determining permanent_uuid...
03:27:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:31 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246043, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.86.62@KRBTEST.COM
03:27:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:31 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246050, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.60@KRBTEST.COM for kudu/127.25.86.60@KRBTEST.COM
03:27:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.533680 25944 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
03:27:31.548 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:31.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.550196 26230 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Bootstrap starting.
03:27:31.550 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:27:31.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:27:31.575 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:27:31.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.581497 26230 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Neither blocks nor log segments found. Creating new log.
03:27:31.589 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.86.62"
port: 36625
}
bound_http_address {
host: "127.25.86.62"
port: 39799
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.86.61"
port: 42583
}
bound_http_address {
host: "127.25.86.61"
port: 35315
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.86.60"
port: 37417
}
bound_http_address {
host: "127.25.86.60"
port: 33367
}
}
}
03:27:31.590 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:27:31.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:31.591 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:27:31.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:27:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.601009 26230 log.cc:824] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Log is configured to *not* fsync() on all Append() calls
03:27:31.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.629621 26230 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: No bootstrap required, opened a new log
03:27:31.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:31 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246051, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:31.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:27:31.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:31.702988 25944 mini_kdc.cc:317] Time spent kinit for test-admin: real 0.105s user 0.005s sys 0.041s
03:27:31.703 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:31.706 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc
03:27:31.718 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:27:31.720 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc
03:27:31.723 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:27:31.732 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:27:31.735 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:27:31.736 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:27:31.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.736663 26230 raft_consensus.cc:348] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:31.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.738233 26230 raft_consensus.cc:374] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:27:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.738914 26230 raft_consensus.cc:729] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82440bfc54d4421c9812da6bb7062fb9, State: Initialized, Role: FOLLOWER
03:27:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.740798 26230 consensus_queue.cc:260] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:31.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.750950 26241 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } } }
03:27:31.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.752424 26241 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [sys.catalog]: This master's current role is: FOLLOWER
03:27:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.757087 26230 sys_catalog.cc:564] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [sys.catalog]: configured and running, proceeding with master startup.
03:27:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:31.828140 26182 tablet.cc:2369] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:27:31.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:31 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246051, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:31.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:27:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:31.846141 25944 mini_kdc.cc:317] Time spent kinit for test-admin: real 0.109s user 0.013s sys 0.037s
03:27:31.847 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:31.847 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc
03:27:31.851 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:27:31.852 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.86.62:36625
03:27:31.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:27:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.856532 25944 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 25970
03:27:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:31.907882 26257 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:31.908339 26257 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:31.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:31.923 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.86.60:37417
03:27:31.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:27:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.924899 25944 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 26142
03:27:31.962 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:31.962 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.86.61:42583
03:27:31.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:27:31.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:31.963534 25944 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 26050
03:27:31.999 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:32.000 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.86.62:36625
03:27:32.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:32.003770 25944 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/logs
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/info.pb
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.62:36625
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.62
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39799
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.62.keytab
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.62
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:27:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.417663 26260 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.418470 26260 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:32.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.419399 26260 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:32.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.455926 26260 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:27:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.456310 26260 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.456841 26260 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.457222 26260 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.457568 26260 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.457909 26260 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.458214 26260 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40777/jwks.json
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:32.504688 26260 master_runner.cc:386] Master server non-default flags:
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.62.keytab
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.62
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.62:36625
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/info.pb
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.62
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39799
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/logs
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:32.506338 26260 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:32.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:32.507635 26260 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.532106 26267 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:33.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.532114 26269 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:33.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:32.532934 26266 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:33.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:32.534456 26260 server_base.cc:1034] running on GCE node
03:27:33.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:33.996335 26260 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:34.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:33.999676 26260 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:34.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.003343 26260 hybrid_clock.cc:648] HybridClock initialized: now 1786246054003241 us; error 84 us; skew 500 ppm
03:27:34.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.004428 26260 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:34.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:34 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246054, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.012672 26260 init.cc:377] Logged in from keytab as kudu/127.25.86.62@KRBTEST.COM (short username kudu)
03:27:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.020778 26260 webserver.cc:460] Webserver started at http://127.25.86.62:39799/ using document root <none> and password file <none>
03:27:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.022250 26260 fs_manager.cc:359] Metadata directory not provided
03:27:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.022512 26260 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.036621 26260 fs_manager.cc:709] Time spent opening directory manager: real 0.009s user 0.002s sys 0.007s
03:27:34.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.043773 26279 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.045516 26260 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.006s sys 0.000s
03:27:34.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.046066 26260 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1499c39a1a345968561ec494147b159"
03:27:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:25 on dist-test-slave-77v9"
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.051819 26260 fs_report.cc:383] FS layout report
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/data
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.101094 26260 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.110396 26260 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:34.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.111112 26260 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.210976 26260 rpc_server.cc:307] RPC server started. Bound to: 127.25.86.62:36625
03:27:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.211117 26336 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.86.62:36625 every 8 connection(s)
03:27:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.214061 26260 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/data/info.pb
03:27:34.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.218695 25944 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 26260
03:27:34.219 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:34.219 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.86.60:37417
03:27:34.220 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:27:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.222054 25944 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/logs
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/info.pb
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.60:37417
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.60
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33367
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.60.keytab
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.60
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:27:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.230104 26337 sys_catalog.cc:263] Verifying existing consensus state
03:27:34.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 03:27:34.231107 26337 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:27:34.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.231680 26260 master.cc:537] Master@127.25.86.62:36625 shutting down...
03:27:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.311561 26260 master.cc:552] Master@127.25.86.62:36625 shutdown complete.
03:27:34.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:27:34.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.629136 26339 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.629899 26339 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.630636 26339 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:34.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.665418 26339 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:27:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.665723 26339 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.666177 26339 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.666529 26339 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.666850 26339 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.667176 26339 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.667409 26339 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40777/jwks.json
03:27:34.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.718047 26339 master_runner.cc:386] Master server non-default flags:
03:27:34.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:34.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:34.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:34.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:34.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.60.keytab
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.60
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.60:37417
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/info.pb
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.60
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33367
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/logs
03:27:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.719616 26339 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:34.721096 26339 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.741427 26345 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.742373 26348 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:34.741493 26346 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.202190 26347 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:27:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.202344 26339 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:27:36.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.206344 26339 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.209352 26339 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.210891 26339 hybrid_clock.cc:648] HybridClock initialized: now 1786246056210831 us; error 69 us; skew 500 ppm
03:27:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.211850 26339 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:36.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:36 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246056, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.223604 26339 init.cc:377] Logged in from keytab as kudu/127.25.86.60@KRBTEST.COM (short username kudu)
03:27:36.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.230008 26339 webserver.cc:460] Webserver started at http://127.25.86.60:33367/ using document root <none> and password file <none>
03:27:36.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.231006 26339 fs_manager.cc:359] Metadata directory not provided
03:27:36.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.231230 26339 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:36.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.240345 26339 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.005s
03:27:36.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.245656 26357 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:36.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.247149 26339 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
03:27:36.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.247602 26339 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:36.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82440bfc54d4421c9812da6bb7062fb9"
03:27:36.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:30 on dist-test-slave-77v9"
03:27:36.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.250869 26339 fs_report.cc:383] FS layout report
03:27:36.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:36.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:36.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/wal
03:27:36.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/data
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.296934 26339 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:36.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.304081 26339 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:36.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.304677 26339 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:36.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.396699 26339 rpc_server.cc:307] RPC server started. Bound to: 127.25.86.60:37417
03:27:36.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.396901 26415 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.86.60:37417 every 8 connection(s)
03:27:36.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.399995 26339 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-2/data/info.pb
03:27:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.406972 25944 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 26339
03:27:36.408 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:36.408 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.86.61:42583
03:27:36.408 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.410846 25944 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/logs
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/info.pb
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:27:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.61:42583
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.61
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35315
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.61.keytab
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.61
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:27:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.416426 26416 sys_catalog.cc:263] Verifying existing consensus state
03:27:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.423116 26416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Bootstrap starting.
03:27:36.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.464537 26416 log.cc:824] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Log is configured to *not* fsync() on all Append() calls
03:27:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.471830 26416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:27:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.472841 26416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Bootstrap complete.
03:27:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.508116 26416 raft_consensus.cc:348] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.511950 26416 raft_consensus.cc:374] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:27:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.512491 26416 raft_consensus.cc:729] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82440bfc54d4421c9812da6bb7062fb9, State: Initialized, Role: FOLLOWER
03:27:36.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.513955 26416 consensus_queue.cc:260] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.517705 26420 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } } }
03:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.519038 26416 sys_catalog.cc:564] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [sys.catalog]: configured and running, proceeding with master startup.
03:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.519050 26420 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [sys.catalog]: This master's current role is: FOLLOWER
03:27:36.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.556257 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:36.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.557149 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:36.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.561276 26364 tablet.cc:2369] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:27:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.552183 26339 master_runner.cc:186] Error getting master registration for 127.25.86.62:36625: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111),
03:27:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.591382 26339 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:27:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.818495 26418 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:27:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.819093 26418 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:27:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.819757 26418 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:27:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.854758 26418 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:27:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.855026 26418 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:27:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.855291 26418 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:27:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.855592 26418 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:27:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.855803 26418 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:27:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.856006 26418 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:27:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.856173 26418 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40777/jwks.json
03:27:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.905134 26418 master_runner.cc:386] Master server non-default flags:
03:27:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:27:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.86.20:44397
03:27:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:27:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:27:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data
03:27:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.86.62:36625,127.25.86.61:42583,127.25.86.60:37417
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/krb5kdc/kudu_127.25.86.61.keytab
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.86.61
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.86.61:42583
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40777/jwks.json
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/info.pb
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.86.61
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35315
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/logs
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:27:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 02:34:27 UTC on ccycloud.quasar-tyqzwd.root.comops.site
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575766
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:27:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.906715 26418 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:27:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.908232 26418 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:27:36.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.940035 26443 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:37.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.997804 26420 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:27:37.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:36.998628 26420 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:37.006632 26361 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:37.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:37.017261 26420 leader_election.cc:288] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1499c39a1a345968561ec494147b159 (127.25.86.62:36625), c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583)
03:27:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:37.040148 26361 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d1499c39a1a345968561ec494147b159 (127.25.86.62:36625): Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:37.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:37.057731 26359 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583): Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:37.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:37.058689 26359 leader_election.cc:302] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82440bfc54d4421c9812da6bb7062fb9; no voters: c804196449a449919c8f1b9437fde2b1, d1499c39a1a345968561ec494147b159
03:27:37.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:37.060688 26420 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:27:37.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:37.562354 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:37.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:37.562733 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:38.567117 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:38.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:38.568322 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:38.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:38.776648 26452 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:27:38.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:38.778793 26452 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:38.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:38.791709 26452 leader_election.cc:288] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1499c39a1a345968561ec494147b159 (127.25.86.62:36625), c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583)
03:27:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:38.803723 26359 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583): Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:38.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:38.809155 26361 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d1499c39a1a345968561ec494147b159 (127.25.86.62:36625): Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:38.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:38.809974 26361 leader_election.cc:302] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82440bfc54d4421c9812da6bb7062fb9; no voters: c804196449a449919c8f1b9437fde2b1, d1499c39a1a345968561ec494147b159
03:27:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:38.811389 26452 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:27:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:38.328671 26442 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 26418
03:27:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:36.941112 26444 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:39.402818 26442 kernel_stack_watchdog.cc:198] Thread 26418 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 392ms:
03:27:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:27:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:27:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:27:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:27:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:39.405180 26418 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.473s user 0.789s sys 1.228s
03:27:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:39.405736 26418 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.474s user 0.790s sys 1.229s
03:27:39.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:39.406864 26445 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2470 milliseconds
03:27:39.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:39.407568 26446 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:27:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.407521 26418 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:27:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.413775 26418 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:27:39.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.417568 26418 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:27:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.419109 26418 hybrid_clock.cc:648] HybridClock initialized: now 1786246059418924 us; error 201 us; skew 500 ppm
03:27:39.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.420244 26418 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:27:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:39 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246059, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.429901 26418 init.cc:377] Logged in from keytab as kudu/127.25.86.61@KRBTEST.COM (short username kudu)
03:27:39.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.438879 26418 webserver.cc:460] Webserver started at http://127.25.86.61:35315/ using document root <none> and password file <none>
03:27:39.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.440137 26418 fs_manager.cc:359] Metadata directory not provided
03:27:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.440475 26418 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:27:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.450486 26418 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
03:27:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.455461 26465 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:27:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.457506 26418 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
03:27:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.458200 26418 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data,/tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c804196449a449919c8f1b9437fde2b1"
03:27:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 03:27:27 on dist-test-slave-77v9"
03:27:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.461139 26418 fs_report.cc:383] FS layout report
03:27:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:27:39.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/data
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:27:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:27:39.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.513381 26418 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:27:39.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.522141 26418 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:27:39.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.522707 26418 kserver.cc:163] Server-wide thread pool size limit: 3276
03:27:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:39.574922 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:39.575421 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.620096 26418 rpc_server.cc:307] RPC server started. Bound to: 127.25.86.61:42583
03:27:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.620265 26518 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.86.61:42583 every 8 connection(s)
03:27:39.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.623186 26418 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/data/info.pb
03:27:39.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.628381 25944 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu as pid 26418
03:27:39.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:27:39.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.639362 26519 sys_catalog.cc:263] Verifying existing consensus state
03:27:39.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 03:27:39.640419 26519 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:27:39.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.641960 26418 master.cc:537] Master@127.25.86.61:42583 shutting down...
03:27:39.699 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:39.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:39.746622 26418 master.cc:552] Master@127.25.86.61:42583 shutdown complete.
03:27:39.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQyuJ_i/test-tmp/mini-kudu-cluster1339416701145460223/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:27:39.786 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.86.60:37417(127.25.86.60:37417)] connecting to peer
03:27:39.789 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:39.853 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:39.853 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:39.855 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:39.855 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.25.86.60:37417(127.25.86.60:37417)] Successfully connected to peer
03:27:39.856 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:39.857 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:39.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:39.961 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:27:39.973 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.25.86.60
03:27:40.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:40 dist-test-slave-77v9 krb5kdc[25957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246051, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.86.60@KRBTEST.COM
03:27:40.259 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8f889810, L:/127.0.0.1:49746 - R:/127.25.86.60:37417] using SASL/GSSAPI
03:27:40.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:40.296 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:40.301 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:40.303 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.303 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.303 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.303 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.304 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.307 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:40.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:40.316 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:40.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.323 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:40.335 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:40.337 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:40.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.342 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.344 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:40.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:40.377 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:40.379 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.379 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.384 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.384 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.387 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:40.394 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:40.396 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:40.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.403 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:40.455 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:40.456 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:40.460 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.462 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.462 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.463 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.465 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:40.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:40.497 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:40.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.500 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.503 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:40.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:40.579385 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:40.579689 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:40.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:40.656 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:40.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.660 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:40.662 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.662 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.663 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:40.663 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:41.034 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:41.037 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:41.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:41.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:41.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:41.040 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:41.040 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:41.041 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:41.042 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:41.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:41.474929 26529 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:27:41.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:41.475318 26529 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:41.477085 26529 leader_election.cc:288] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1499c39a1a345968561ec494147b159 (127.25.86.62:36625), c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583)
03:27:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:41.483379 26359 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583): Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:41.483951 26361 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d1499c39a1a345968561ec494147b159 (127.25.86.62:36625): Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:41.484314 26361 leader_election.cc:302] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82440bfc54d4421c9812da6bb7062fb9; no voters: c804196449a449919c8f1b9437fde2b1, d1499c39a1a345968561ec494147b159
03:27:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:41.485008 26529 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:27:41.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:41.582875 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:41.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:41.583159 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:41.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:41.594340 26339 master_runner.cc:186] Error getting master registration for 127.25.86.62:36625: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111),
03:27:41.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:41.594807 26339 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:27:41.954 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:41.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:41.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:41.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:41.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:41.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:41.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:41.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:41.961 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:42.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:42.117 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:42.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:42.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:42.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:42.122 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:42.124 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:42.124 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:42.124 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:42.586340 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:42.586676 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:42.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:42.756 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:42.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:42.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:42.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:42.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:42.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:42.760 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:42.763 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:43.589916 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:43.590205 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:44.593549 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:44.593904 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:45.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:45.598881 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:45.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:45.599301 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:46 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246066, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:46.228377 26353 init.cc:295] Successfully reacquired a new kerberos TGT
03:27:46.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:46.248852 26539 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:27:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:46.249739 26539 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:46.252081 26539 leader_election.cc:288] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1499c39a1a345968561ec494147b159 (127.25.86.62:36625), c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583)
03:27:46.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:46.256270 26359 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111) [suppressed 6 similar messages]
03:27:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:46.259431 26361 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d1499c39a1a345968561ec494147b159 (127.25.86.62:36625): Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:46.259945 26359 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583): Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:46.260267 26359 leader_election.cc:302] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82440bfc54d4421c9812da6bb7062fb9; no voters: c804196449a449919c8f1b9437fde2b1, d1499c39a1a345968561ec494147b159
03:27:46.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:46.261058 26539 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:27:46.535 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:46.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:46.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:46.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:46.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:46.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:46.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:46.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:46.543 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:46.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:46.597465 26339 master_runner.cc:186] Error getting master registration for 127.25.86.62:36625: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111),
03:27:46.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:46.597956 26339 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:27:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:46.602834 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:46.603116 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:47.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:47.606451 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:47.606714 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:48.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:48.610167 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:48.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:48.610522 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:49.613729 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:49.614017 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:50.053078 25414 debug-util.cc:398] Leaking SignalData structure 0x7b08000919a0 after lost signal to thread 25341
03:27:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:50.053993 25414 debug-util.cc:398] Leaking SignalData structure 0x7b080003e7a0 after lost signal to thread 25417
03:27:50.515 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:50.517 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:50.519 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:50.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:50.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:50.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:50.520 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:50.520 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:50.522 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:50.618892 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:50.619329 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:51.281651 26548 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:27:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:51.282137 26548 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:51.283993 26548 leader_election.cc:288] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1499c39a1a345968561ec494147b159 (127.25.86.62:36625), c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583)
03:27:51.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:51.287738 26361 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:51.291501 26359 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583): Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:51.292222 26361 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d1499c39a1a345968561ec494147b159 (127.25.86.62:36625): Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:51.292604 26361 leader_election.cc:302] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82440bfc54d4421c9812da6bb7062fb9; no voters: c804196449a449919c8f1b9437fde2b1, d1499c39a1a345968561ec494147b159
03:27:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:51.293365 26548 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:27:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:51.600798 26339 master_runner.cc:186] Error getting master registration for 127.25.86.62:36625: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111),
03:27:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:51.601269 26339 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:27:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:51.623023 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:51.623350 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:52.626753 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:52.627077 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:53.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:53.631700 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:53.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:53.632032 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:53.654 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:53.657 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:53.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:53.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:53.659 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:53.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:53.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:53.660 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:53.661 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:53.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:53.905547 25622 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd60 after lost signal to thread 25490
03:27:53.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:53.906844 25622 debug-util.cc:398] Leaking SignalData structure 0x7b08000af4a0 after lost signal to thread 25626
03:27:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:54.639927 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:54.640343 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:55.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:55.471423 26555 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:27:55.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:55.471980 26555 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:55.473997 26555 leader_election.cc:288] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1499c39a1a345968561ec494147b159 (127.25.86.62:36625), c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583)
03:27:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:55.497915 26359 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583): Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:55.498550 26361 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d1499c39a1a345968561ec494147b159 (127.25.86.62:36625): Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:55.499096 26361 leader_election.cc:302] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82440bfc54d4421c9812da6bb7062fb9; no voters: c804196449a449919c8f1b9437fde2b1, d1499c39a1a345968561ec494147b159
03:27:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:55.500355 26555 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:27:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:55.643523 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:55.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:55.643870 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:56.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:27:56 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246076, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:27:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:56.231282 26353 init.cc:295] Successfully reacquired a new kerberos TGT
03:27:56.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:56.605515 26339 master_runner.cc:186] Error getting master registration for 127.25.86.62:36625: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111),
03:27:56.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:56.606215 26339 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:27:56.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:56.646972 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:56.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:56.647403 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:56.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:56.736974 25244 debug-util.cc:398] Leaking SignalData structure 0x7b080009b0a0 after lost signal to thread 25177
03:27:56.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:56.738000 25244 debug-util.cc:398] Leaking SignalData structure 0x7b080009da60 after lost signal to thread 25248
03:27:57.554 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:27:57.556 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:27:57.561 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:57.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:57.562 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:27:57.562 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:57.562 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:57.562 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:27:57.562 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:27:57.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:57.647994 26563 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:27:57.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:57.648597 26563 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:27:57.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:57.651443 26563 leader_election.cc:288] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1499c39a1a345968561ec494147b159 (127.25.86.62:36625), c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583)
03:27:57.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:57.652218 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:57.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:57.653587 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:57.653839 26361 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:27:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:57.658447 26361 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d1499c39a1a345968561ec494147b159 (127.25.86.62:36625): Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:27:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:57.659258 26359 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583): Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:27:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:57.659708 26359 leader_election.cc:302] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82440bfc54d4421c9812da6bb7062fb9; no voters: c804196449a449919c8f1b9437fde2b1, d1499c39a1a345968561ec494147b159
03:27:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:27:57.660545 26563 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:27:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:58.656725 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:58.657204 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:27:59.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:59.660303 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:27:59.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:27:59.660630 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:00.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:28:00.117 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:28:00.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:00.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:00.120 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:00.120 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:00.120 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:00.121 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:00.122 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:28:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:00.664275 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:28:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:00.664615 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:01.609429 26339 master_runner.cc:186] Error getting master registration for 127.25.86.62:36625: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111),
03:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:01.609906 26339 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:28:01.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:01.668056 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:28:01.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:01.668359 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:02.672125 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:28:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:02.672405 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:03.534 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:28:03.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:28:03.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:03.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:03.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:03.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:03.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:03.541 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:03.542 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:28:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:03.676874 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:28:03.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:03.677199 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:03.715 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:28:03.717 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:28:03.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:03.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:03.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:03.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:03.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:03.720 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:03.721 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:28:04.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:04.158952 25622 debug-util.cc:398] Leaking SignalData structure 0x7b080003eee0 after lost signal to thread 25490
03:28:04.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:04.160157 25622 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9f80 after lost signal to thread 25626
03:28:04.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:04.682086 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:28:04.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:04.682490 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:05.685535 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:28:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:05.685851 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:05.795 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:28:05.796 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:28:05.797 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:05.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:05.797 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:05.797 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:05.797 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:05.797 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:05.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:28:06.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 03:28:06 dist-test-slave-77v9 krb5kdc[25957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786246086, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.86.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:28:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:06.234431 26353 init.cc:295] Successfully reacquired a new kerberos TGT
03:28:06.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:06.613695 26339 master_runner.cc:186] Error getting master registration for 127.25.86.62:36625: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111),
03:28:06.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:06.614221 26339 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:28:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:06.688987 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:28:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:06.689280 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:06.751327 26580 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:28:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:06.752002 26580 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1499c39a1a345968561ec494147b159" member_type: VOTER last_known_addr { host: "127.25.86.62" port: 36625 } } peers { permanent_uuid: "c804196449a449919c8f1b9437fde2b1" member_type: VOTER last_known_addr { host: "127.25.86.61" port: 42583 } } peers { permanent_uuid: "82440bfc54d4421c9812da6bb7062fb9" member_type: VOTER last_known_addr { host: "127.25.86.60" port: 37417 } }
03:28:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:06.754626 26580 leader_election.cc:288] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1499c39a1a345968561ec494147b159 (127.25.86.62:36625), c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583)
03:28:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:06.756058 26361 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:28:06.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:06.760294 26361 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d1499c39a1a345968561ec494147b159 (127.25.86.62:36625): Network error: Client connection negotiation failed: client connection to 127.25.86.62:36625: connect: Connection refused (error 111)
03:28:06.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:06.760895 26359 leader_election.cc:334] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c804196449a449919c8f1b9437fde2b1 (127.25.86.61:42583): Network error: Client connection negotiation failed: client connection to 127.25.86.61:42583: connect: Connection refused (error 111)
03:28:06.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:06.761364 26359 leader_election.cc:302] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82440bfc54d4421c9812da6bb7062fb9; no voters: c804196449a449919c8f1b9437fde2b1, d1499c39a1a345968561ec494147b159
03:28:06.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:06.762121 26580 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:28:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:07.696506 26433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82440bfc54d4421c9812da6bb7062fb9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:28:07.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:07.697440 26433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:28:08.554 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.62:36625(127.25.86.62:36625)] connecting to peer
03:28:08.555 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.86.61:42583(127.25.86.61:42583)] connecting to peer
03:28:08.556 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:08.556 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:08.557 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.62:36625(127.25.86.62:36625)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:08.557 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.86.61:42583(127.25.86.61:42583)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:08.557 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.61:42583: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583
03:28:08.557 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.86.62:36625: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
03:28:08.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583; will retry
03:28:08.563 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28927), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [217ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [220ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [636ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [657ms] refreshing cache from master, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [666ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [667ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [669ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [670ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [682ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [683ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [685ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [704ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [706ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [743ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [750ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [751ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [762ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [763ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [766ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [825ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [828ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [857ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [865ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1025ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1026ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1405ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1406ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2317ms] refreshing cache from master, [2317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2323ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2324ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2477ms] refreshing cache from master, [2478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2482ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2484ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2487ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2487ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [3117ms] refreshing cache from master, [3117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [3122ms] too many traces: truncated at 100 traces, deferred=Deferred@712851173(state=PENDING, result=null, callback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646), errback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646)))
03:28:08.569 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.86.62:36625(127.25.86.62:36625)] disconnecting while in state TERMINATED
03:28:08.571 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.86.60:37417(127.25.86.60:37417)] disconnecting while in state READY
03:28:08.571 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.86.61:42583(127.25.86.61:42583)] disconnecting while in state TERMINATED
03:28:08.579 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:28:08.581 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.25.86.60:37417(127.25.86.60:37417)] handling channelInactive
03:28:08.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.86.60:37417(127.25.86.60:37417)] cleaning up while in state READY due to: connection closed
03:28:08.583 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:28:08.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:08.591307 25944 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 26339
03:28:08.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:08.715787 25622 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9f20 after lost signal to thread 25490
03:28:08.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 03:28:08.717772 25622 debug-util.cc:398] Leaking SignalData structure 0x7b080003f060 after lost signal to thread 25626
03:28:09.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T03:28:09Z chronyd exiting
03:28:10.962 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:28:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:10.964553 25163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 25489
03:28:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:11.005959 25163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 25634
03:28:11.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:11.043799 25163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 25792
03:28:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:11.088022 25163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 25176
03:28:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:11.134928 25163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 25251
03:28:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 03:28:11.173219 25163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQyuJ_i/build/tsan/bin/kudu with pid 25333
03:28:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T03:28:12Z chronyd exiting
03:28:13.498 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28927), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [217ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [220ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [636ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [657ms] refreshing cache from master, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [666ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [667ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [669ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [670ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [682ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [683ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [685ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [704ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [706ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [743ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [750ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [751ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [762ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [763ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [766ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [825ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [828ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [857ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [865ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1025ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1026ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1405ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1406ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2317ms] refreshing cache from master, [2317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2323ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2324ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2477ms] refreshing cache from master, [2478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2482ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2484ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2487ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2487ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [3117ms] refreshing cache from master, [3117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [3122ms] too many traces: truncated at 100 traces, deferred=Deferred@712851173(state=PENDING, result=null, callback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646), errback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.86.62:36625
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28927), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [217ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [220ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [636ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [657ms] refreshing cache from master, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [666ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [667ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [669ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [670ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [682ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [683ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [685ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [704ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [706ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [743ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [750ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [751ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [762ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [763ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [766ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [825ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [828ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [857ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [865ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1025ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1026ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1405ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1406ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2317ms] refreshing cache from master, [2317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2323ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2324ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2477ms] refreshing cache from master, [2478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2482ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2484ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2487ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2487ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [3117ms] refreshing cache from master, [3117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [3122ms] too many traces: truncated at 100 traces, deferred=Deferred@712851173(state=PENDING, result=null, callback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646), errback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.86.62:36625
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
03:28:13.977 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:28:13.978 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28927), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [217ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [220ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [636ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [657ms] refreshing cache from master, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [666ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [667ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [669ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [670ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [682ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [683ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [685ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [704ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [706ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [743ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [750ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [751ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [762ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [763ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [766ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [825ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [828ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [857ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [865ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1025ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1026ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1405ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1406ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2317ms] refreshing cache from master, [2317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2323ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2324ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2477ms] refreshing cache from master, [2478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2482ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2484ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2487ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2487ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [3117ms] refreshing cache from master, [3117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [3122ms] too many traces: truncated at 100 traces, deferred=Deferred@712851173(state=PENDING, result=null, callback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646), errback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.86.62:36625
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28927), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [217ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [220ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [636ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [657ms] refreshing cache from master, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [666ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [667ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [669ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [670ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [682ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [683ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [685ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [704ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [706ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [742ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [743ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [750ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [751ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [762ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [763ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [766ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [825ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [828ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [857ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [862ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [865ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1025ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1026ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1026ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [1405ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [1406ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2317ms] refreshing cache from master, [2317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2321ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2323ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2324ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2477ms] refreshing cache from master, [2478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [2480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [2482ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.61:42583: Network error: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583, [2484ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.60:37417: OK, [2487ms] Sub RPC ConnectToMaster: received response from server master-127.25.86.62:36625: Network error: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [2487ms] delaying RPC due to: Service unavailable: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625, [3117ms] refreshing cache from master, [3117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.62:36625, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.60:37417, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.86.61:42583, [3122ms] too many traces: truncated at 100 traces, deferred=Deferred@712851173(state=PENDING, result=null, callback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646), errback=(continuation of Deferred@1129867735 after retry RPC after error@1238364871) -> (continuation of Deferred@1322458672 after retry RPC after error@219847449) -> (continuation of Deferred@227298748 after retry RPC after error@1004789504) -> (continuation of Deferred@1838638290 after retry RPC after error@873265194) -> (continuation of Deferred@1535292090 after retry RPC after error@2126282005) -> (continuation of Deferred@685455332 after retry RPC after error@2068137853) -> (continuation of Deferred@891039181 after retry RPC after error@685095598) -> (continuation of Deferred@1505807431 after retry RPC after error@141498029) -> (continuation of Deferred@1548300328 after retry RPC after error@1506599365) -> (continuation of Deferred@103922767 after retry RPC after error@1981487194) -> (continuation of Deferred@1665580737 after retry RPC after error@1572518309) -> (continuation of Deferred@1457694105 after retry RPC after error@2014686935) -> (continuation of Deferred@787724194 after retry RPC after error@638338120) -> (continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.86.62:36625,127.25.86.60:37417,127.25.86.61:42583) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.61:42583(127.25.86.61:42583): Connection refused: /127.25.86.61:42583,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.86.62:36625(127.25.86.62:36625): Connection refused: /127.25.86.62:36625
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.86.62:36625
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)