07:40:39.414 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:40:39.426 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskzm7lNz/test-tmp
07:40:39.429 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:40:39.432 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskzm7lNz/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:40:39.655 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:40:39.912 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:40:39.913 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:40:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:39.917152 10358 mini_oidc.cc:152] Starting JWKS server
07:40:39.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:39.921813 10358 webserver.cc:460] Webserver started at http://127.0.0.1:41575/ using document root <none> and password file <none>
07:40:39.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:39.922371 10358 mini_oidc.cc:189] Starting OIDC Discovery server
07:40:39.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:39.924386 10358 webserver.cc:460] Webserver started at http://127.0.0.1:38173/ using document root <none> and password file <none>
07:40:39.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:40:39Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:40:39.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:40:39Z Disabled control of system clock
07:40:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:39.985399 10358 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/wal
07:40:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/data
07:40:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:40:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:40:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:40:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/logs
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/data/info.pb
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.190:38881
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.190
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:39.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:39.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:39.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:39.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:39.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:39.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41575/jwks.json with env {}
07:40:40.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.295904 10374 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:40:40.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.296654 10374 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:40:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.297376 10374 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:40:40.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.327783 10374 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:40:40.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.328197 10374 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:40:40.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.328540 10374 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:40:40.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.328879 10374 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:40:40.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.329231 10374 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:40:40.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.329501 10374 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41575/jwks.json
07:40:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:40.364825 10374 master_runner.cc:386] Master server non-default flags:
07:40:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:40.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:40.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:40.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:40.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/data
07:40:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/wal
07:40:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.190:38881
07:40:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:40:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41575/jwks.json
07:40:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/data/info.pb
07:40:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.190
07:40:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:40:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:40:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:40:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/logs
07:40:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:40:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:40:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:40:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:40:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:40:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:40:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:40:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:40:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:40:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:40.366684 10374 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:40:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:40.369089 10374 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:40:40.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.391615 10385 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:41.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.391618 10383 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:41.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:40.391868 10382 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:41.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:40.392922 10374 server_base.cc:1034] running on GCE node
07:40:41.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.643046 10374 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:40:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.646767 10374 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:40:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.648283 10374 hybrid_clock.cc:648] HybridClock initialized: now 1786002041648257 us; error 86 us; skew 500 ppm
07:40:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.649231 10374 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:40:41.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.660142 10374 webserver.cc:460] Webserver started at http://127.10.29.190:36791/ using document root <none> and password file <none>
07:40:41.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.661177 10374 fs_manager.cc:359] Metadata directory not provided
07:40:41.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.661362 10374 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:40:41.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.661844 10374 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:40:41.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.666783 10374 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/data/instance:
07:40:41.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54cd480bcfd04419a4e71a3039c90a9e"
07:40:41.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:41 on dist-test-slave-04bb"
07:40:41.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.667970 10374 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/wal/instance:
07:40:41.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54cd480bcfd04419a4e71a3039c90a9e"
07:40:41.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:41 on dist-test-slave-04bb"
07:40:41.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.675662 10374 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
07:40:41.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.681797 10395 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:40:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.683060 10374 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
07:40:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.683358 10374 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/wal
07:40:41.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54cd480bcfd04419a4e71a3039c90a9e"
07:40:41.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:41 on dist-test-slave-04bb"
07:40:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.683640 10374 fs_report.cc:383] FS layout report
07:40:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:40:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/wal
07:40:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/wal
07:40:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/data/data
07:40:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:40:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:40:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:40:41.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.728092 10374 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:40:41.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.729700 10374 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:40:41.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.730100 10374 kserver.cc:163] Server-wide thread pool size limit: 3276
07:40:41.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.813892 10374 rpc_server.cc:307] RPC server started. Bound to: 127.10.29.190:38881
07:40:41.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.813973 10471 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.29.190:38881 every 8 connection(s)
07:40:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.817102 10374 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/data/info.pb
07:40:41.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.825109 10474 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:40:41.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.827358 10358 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 10374
07:40:41.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.827804 10358 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-0/wal/instance
07:40:41.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.831189 10358 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:41.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:41.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/wal
07:40:41.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/data
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/logs
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/data/info.pb
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:41.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.189:45741
07:40:41.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.189
07:40:41.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:41.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:41.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:41.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:41.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:41.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:41.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:41.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41575/jwks.json with env {}
07:40:41.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.841498 10474 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } has no permanent_uuid. Determining permanent_uuid...
07:40:41.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.877668 10474 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } has no permanent_uuid. Determining permanent_uuid...
07:40:41.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:41.880448 10396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.29.189:45741: connect: Connection refused (error 111)
07:40:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:41.885190 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.189:45741: Network error: Client connection negotiation failed: client connection to 127.10.29.189:45741: connect: Connection refused (error 111)
07:40:41.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:41.934692 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } attempt: 1
07:40:41.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:41.939136 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.189:45741: Network error: Client connection negotiation failed: client connection to 127.10.29.189:45741: connect: Connection refused (error 111)
07:40:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:42.007548 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } attempt: 2
07:40:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.011485 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.189:45741: Network error: Client connection negotiation failed: client connection to 127.10.29.189:45741: connect: Connection refused (error 111)
07:40:42.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:42.102859 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } attempt: 3
07:40:42.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.106678 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.189:45741: Network error: Client connection negotiation failed: client connection to 127.10.29.189:45741: connect: Connection refused (error 111)
07:40:42.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.159112 10478 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:40:42.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.159664 10478 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:40:42.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.160274 10478 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:40:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.191073 10478 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:40:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.191334 10478 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:40:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.191521 10478 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:40:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.191717 10478 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:40:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.191892 10478 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:40:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.192019 10478 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41575/jwks.json
07:40:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:42.227099 10478 master_runner.cc:386] Master server non-default flags:
07:40:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/data
07:40:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/wal
07:40:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.189:45741
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41575/jwks.json
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/data/info.pb
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.189
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/logs
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:40:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:40:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:40:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:40:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:40:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:40:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:40:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:40:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:40:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:42.228364 10478 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:40:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:42.230497 10478 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:40:42.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.248852 10493 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:42.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:42.250115 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } attempt: 4
07:40:42.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.253880 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.189:45741: Network error: Client connection negotiation failed: client connection to 127.10.29.189:45741: connect: Connection refused (error 111)
07:40:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:42.553339 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } attempt: 5
07:40:42.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.558601 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.189:45741: Network error: Client connection negotiation failed: client connection to 127.10.29.189:45741: connect: Connection refused (error 111)
07:40:43.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.106226 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } attempt: 6
07:40:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.114317 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.189:45741: Network error: Client connection negotiation failed: client connection to 127.10.29.189:45741: connect: Connection refused (error 111)
07:40:43.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.248944 10490 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:43.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:42.249768 10489 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:43.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.458277 10478 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:40:43.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.458835 10491 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:40:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.462643 10478 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:40:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.466213 10478 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:40:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.467685 10478 hybrid_clock.cc:648] HybridClock initialized: now 1786002043467637 us; error 67 us; skew 500 ppm
07:40:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.468639 10478 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:40:43.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.475662 10478 webserver.cc:460] Webserver started at http://127.10.29.189:45007/ using document root <none> and password file <none>
07:40:43.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.476815 10478 fs_manager.cc:359] Metadata directory not provided
07:40:43.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.477029 10478 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:40:43.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.477460 10478 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:40:43.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.482774 10478 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/data/instance:
07:40:43.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd0b61b1e14747358d6130a18fea1817"
07:40:43.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:43 on dist-test-slave-04bb"
07:40:43.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.483978 10478 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/wal/instance:
07:40:43.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd0b61b1e14747358d6130a18fea1817"
07:40:43.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:43 on dist-test-slave-04bb"
07:40:43.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.492026 10478 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
07:40:43.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.497860 10504 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:40:43.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.498910 10478 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
07:40:43.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.499287 10478 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/wal
07:40:43.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd0b61b1e14747358d6130a18fea1817"
07:40:43.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:43 on dist-test-slave-04bb"
07:40:43.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.499693 10478 fs_report.cc:383] FS layout report
07:40:43.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/wal
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/wal
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/data/data
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:40:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:40:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:40:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:40:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:40:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:40:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:40:43.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.566969 10478 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:40:43.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.568819 10478 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:40:43.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.569396 10478 kserver.cc:163] Server-wide thread pool size limit: 3276
07:40:43.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.653321 10478 rpc_server.cc:307] RPC server started. Bound to: 127.10.29.189:45741
07:40:43.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.653398 10569 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.29.189:45741 every 8 connection(s)
07:40:43.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.656404 10478 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/data/info.pb
07:40:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.657397 10358 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 10478
07:40:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.657811 10358 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-1/wal/instance
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.661090 10358 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/wal
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/data
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/logs
07:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/data/info.pb
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.188:43795
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.188
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41575/jwks.json with env {}
07:40:43.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.663463 10571 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:40:43.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.677126 10571 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } has no permanent_uuid. Determining permanent_uuid...
07:40:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.701138 10571 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } has no permanent_uuid. Determining permanent_uuid...
07:40:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.719156 10571 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } has no permanent_uuid. Determining permanent_uuid...
07:40:43.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.722262 10506 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.726925 10571 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:43.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.776398 10571 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 1
07:40:43.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.780244 10571 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.848685 10571 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 2
07:40:43.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.852651 10571 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:43.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:43.944082 10571 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 3
07:40:43.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.947929 10571 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.995690 10573 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:40:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.996381 10573 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:40:43.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:43.997349 10573 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:40:44.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.027876 10573 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:40:44.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.028139 10573 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:40:44.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.028416 10573 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:40:44.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.028728 10573 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:40:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.028939 10573 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:40:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.029080 10573 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41575/jwks.json
07:40:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.063244 10573 master_runner.cc:386] Master server non-default flags:
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/data
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/wal
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.188:43795
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41575/jwks.json
07:40:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/data/info.pb
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.188
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/logs
07:40:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:40:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:40:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:40:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:40:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:40:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:40:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:40:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:40:44.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:40:44.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.064445 10573 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:40:44.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.066397 10573 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:40:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.086282 10591 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.091315 10571 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 4
07:40:44.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.096634 10571 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.174960 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } attempt: 7
07:40:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.190150 10474 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } has no permanent_uuid. Determining permanent_uuid...
07:40:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.195602 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.254024 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 1
07:40:44.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.258667 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:44.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.340086 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 2
07:40:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.344945 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.396111 10571 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 5
07:40:44.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.400378 10571 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:44.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.430342 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 3
07:40:44.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.435027 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.575472 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 4
07:40:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.580267 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:44.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.863685 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 5
07:40:44.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.868491 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:44.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:44.948074 10571 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 6
07:40:44.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.955677 10571 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.086336 10595 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:44.086781 10589 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:45.357388 10593 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:40:45.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.357733 10573 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:40:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.361722 10573 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:40:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.364403 10573 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:40:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.365839 10573 hybrid_clock.cc:648] HybridClock initialized: now 1786002045365790 us; error 66 us; skew 500 ppm
07:40:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.366750 10573 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:40:45.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.374356 10573 webserver.cc:460] Webserver started at http://127.10.29.188:43195/ using document root <none> and password file <none>
07:40:45.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.375425 10573 fs_manager.cc:359] Metadata directory not provided
07:40:45.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.375674 10573 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:40:45.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.376255 10573 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:40:45.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.381213 10573 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/data/instance:
07:40:45.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8"
07:40:45.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:45 on dist-test-slave-04bb"
07:40:45.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.382503 10573 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/wal/instance:
07:40:45.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8"
07:40:45.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:45 on dist-test-slave-04bb"
07:40:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.389911 10573 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
07:40:45.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.395429 10607 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:40:45.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.396404 10573 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
07:40:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.396809 10573 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/wal
07:40:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8"
07:40:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:45 on dist-test-slave-04bb"
07:40:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.397243 10573 fs_report.cc:383] FS layout report
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/wal
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/wal
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/data/data
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:40:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:40:45.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.421087 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 6
07:40:45.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:45.427996 10474 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.29.188:43795: Network error: Client connection negotiation failed: client connection to 127.10.29.188:43795: connect: Connection refused (error 111)
07:40:45.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.432389 10573 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:40:45.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.434284 10573 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:40:45.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.434854 10573 kserver.cc:163] Server-wide thread pool size limit: 3276
07:40:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.512207 10573 rpc_server.cc:307] RPC server started. Bound to: 127.10.29.188:43795
07:40:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.512312 10695 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.29.188:43795 every 8 connection(s)
07:40:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.515038 10573 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/data/info.pb
07:40:45.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.518098 10358 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 10573
07:40:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.518671 10358 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/master-2/wal/instance
07:40:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.521106 10696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:40:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.537777 10696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } has no permanent_uuid. Determining permanent_uuid...
07:40:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.564467 10696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } has no permanent_uuid. Determining permanent_uuid...
07:40:45.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.577720 10696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } has no permanent_uuid. Determining permanent_uuid...
07:40:45.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.597473 10696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: Bootstrap starting.
07:40:45.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.604722 10696 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: Neither blocks nor log segments found. Creating new log.
07:40:45.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.607227 10696 log.cc:824] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: Log is configured to *not* fsync() on all Append() calls
07:40:45.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.611955 10696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: No bootstrap required, opened a new log
07:40:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.633611 10696 raft_consensus.cc:348] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.634235 10696 raft_consensus.cc:374] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:40:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.634418 10696 raft_consensus.cc:729] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aa4b4ac2ec7a4b03a45d3c1a1b1a66a8, State: Initialized, Role: FOLLOWER
07:40:45.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.635221 10696 consensus_queue.cc:260] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:45.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.638370 10707 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:45.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.639564 10707 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: This master's current role is: FOLLOWER
07:40:45.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.640291 10696 sys_catalog.cc:564] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: configured and running, proceeding with master startup.
07:40:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:45.669147 10721 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:40:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:45.672559 10721 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:40:45.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:45.691408 10623 tablet.cc:2369] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:40:45.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.664229 10573 master_runner.cc:186] Error getting master registration for 127.10.29.190:38881: OK, instance_id { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" instance_seqno: 1786002041787585 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:40:45.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:45.692989 10573 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:40:46.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.016283 10571 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 7
07:40:46.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.035661 10571 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: Bootstrap starting.
07:40:46.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.041177 10571 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: Neither blocks nor log segments found. Creating new log.
07:40:46.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.043080 10571 log.cc:824] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: Log is configured to *not* fsync() on all Append() calls
07:40:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.047768 10571 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: No bootstrap required, opened a new log
07:40:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.069489 10571 raft_consensus.cc:348] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.070096 10571 raft_consensus.cc:374] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:40:46.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.070277 10571 raft_consensus.cc:729] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd0b61b1e14747358d6130a18fea1817, State: Initialized, Role: FOLLOWER
07:40:46.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.070990 10571 consensus_queue.cc:260] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.073779 10726 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.074661 10726 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [sys.catalog]: This master's current role is: FOLLOWER
07:40:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.075510 10571 sys_catalog.cc:564] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [sys.catalog]: configured and running, proceeding with master startup.
07:40:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:46.077661 10510 tablet.cc:2369] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:40:46.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.106251 10707 raft_consensus.cc:482] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:40:46.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.106802 10707 raft_consensus.cc:504] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:46.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.111919 10538 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dd0b61b1e14747358d6130a18fea1817" is_pre_election: true
07:40:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.112874 10707 leader_election.cc:288] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 54cd480bcfd04419a4e71a3039c90a9e (127.10.29.190:38881), dd0b61b1e14747358d6130a18fea1817 (127.10.29.189:45741)
07:40:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.113081 10538 raft_consensus.cc:2454] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 in term 0.
07:40:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.115032 10608 leader_election.cc:302] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aa4b4ac2ec7a4b03a45d3c1a1b1a66a8, dd0b61b1e14747358d6130a18fea1817; no voters:
07:40:46.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.116139 10707 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:40:46.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.116680 10707 raft_consensus.cc:482] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:40:46.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.117792 10707 raft_consensus.cc:3037] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 0 FOLLOWER]: Advancing to term 1
07:40:46.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.125649 10707 raft_consensus.cc:504] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:46.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.137878 10707 leader_election.cc:288] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [CANDIDATE]: Term 1 election: Requested vote from peers 54cd480bcfd04419a4e71a3039c90a9e (127.10.29.190:38881), dd0b61b1e14747358d6130a18fea1817 (127.10.29.189:45741)
07:40:46.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.138059 10538 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dd0b61b1e14747358d6130a18fea1817"
07:40:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.138653 10538 raft_consensus.cc:3037] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [term 0 FOLLOWER]: Advancing to term 1
07:40:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:46.139457 10739 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:40:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:46.139793 10739 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:40:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.151041 10538 raft_consensus.cc:2454] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 in term 1.
07:40:46.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.152635 10608 leader_election.cc:302] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aa4b4ac2ec7a4b03a45d3c1a1b1a66a8, dd0b61b1e14747358d6130a18fea1817; no voters:
07:40:46.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.153539 10707 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 1 FOLLOWER]: Leader election won for term 1
07:40:46.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.165390 10707 raft_consensus.cc:686] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [term 1 LEADER]: Becoming Leader. State: Replica: aa4b4ac2ec7a4b03a45d3c1a1b1a66a8, State: Running, Role: LEADER
07:40:46.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.166622 10707 consensus_queue.cc:237] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.185943 10743 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa4b4ac2ec7a4b03a45d3c1a1b1a66a8. Latest consensus state: current_term: 1 leader_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:46.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.189257 10743 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: This master's current role is: LEADER
07:40:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.125335 10478 master_runner.cc:186] Error getting master registration for 127.10.29.190:38881: OK, instance_id { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" instance_seqno: 1786002041787585 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:40:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.208770 10478 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:40:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.210130 10745 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:40:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.217756 10745 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:40:46.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.240873 10538 raft_consensus.cc:1261] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [term 1 FOLLOWER]: Refusing update from remote peer aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:40:46.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.242681 10707 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:40:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.268883 10726 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa4b4ac2ec7a4b03a45d3c1a1b1a66a8. Latest consensus state: current_term: 1 leader_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.270128 10748 mvcc.cc:205] Tried to move back new op lower bound from 7315464381383581696 to 7315464381121155072. Current Snapshot: MvccSnapshot[applied={T|T < 7315464381383581696}]
07:40:46.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.270957 10726 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [sys.catalog]: This master's current role is: FOLLOWER
07:40:46.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.273710 10707 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:46.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.274621 10707 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: This master's current role is: LEADER
07:40:46.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.276278 10745 catalog_manager.cc:1347] Generated new cluster ID: c328fd767aef487197fe7a5d3be0ea84
07:40:46.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.276602 10745 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:40:46.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.291225 10726 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.294355 10726 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817 [sys.catalog]: This master's current role is: FOLLOWER
07:40:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.324697 10745 catalog_manager.cc:1370] Generated new certificate authority record
07:40:46.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.327396 10745 catalog_manager.cc:1504] Loading token signing keys...
07:40:46.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.356267 10745 catalog_manager.cc:5868] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: Generated new TSK 0
07:40:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.360064 10745 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:40:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.394964 10358 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/wal
07:40:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/data
07:40:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:40:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:40:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/logs
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/data/info.pb
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.129:0
07:40:46.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.29.129
07:40:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.129
07:40:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:46.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:40:46.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.461377 10474 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } attempt: 7
07:40:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.510046 10474 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e: Bootstrap starting.
07:40:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.529237 10474 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e: Neither blocks nor log segments found. Creating new log.
07:40:46.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.535481 10474 log.cc:824] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e: Log is configured to *not* fsync() on all Append() calls
07:40:46.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.571935 10474 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e: No bootstrap required, opened a new log
07:40:46.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.654939 10474 raft_consensus.cc:348] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.657472 10474 raft_consensus.cc:374] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:40:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.658516 10474 raft_consensus.cc:729] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 54cd480bcfd04419a4e71a3039c90a9e, State: Initialized, Role: FOLLOWER
07:40:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.661049 10474 consensus_queue.cc:260] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } }
07:40:46.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.685835 10430 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "54cd480bcfd04419a4e71a3039c90a9e" is_pre_election: true
07:40:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.687065 10429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "54cd480bcfd04419a4e71a3039c90a9e"
07:40:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.691190 10430 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 in term 0.
07:40:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.698484 10428 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [term 0 FOLLOWER]: Advancing to term 1
07:40:46.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.705865 10428 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [term 1 FOLLOWER]: Refusing update from remote peer aa4b4ac2ec7a4b03a45d3c1a1b1a66a8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:40:46.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.709841 10707 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:40:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.715201 10762 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.716536 10762 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [sys.catalog]: This master's current role is: FOLLOWER
07:40:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.716135 10474 sys_catalog.cc:564] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [sys.catalog]: configured and running, proceeding with master startup.
07:40:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:46.742630 10402 tablet.cc:2369] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:40:46.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.755821 10767 mvcc.cc:205] Tried to move back new op lower bound from 7315464381383581696 to 7315464381121155072. Current Snapshot: MvccSnapshot[applied={T|T < 7315464381383581696}]
07:40:46.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.778355 10762 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa4b4ac2ec7a4b03a45d3c1a1b1a66a8. Latest consensus state: current_term: 1 leader_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:46.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.779088 10762 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [sys.catalog]: This master's current role is: FOLLOWER
07:40:46.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.780823 10763 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:46.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.781597 10763 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e [sys.catalog]: This master's current role is: FOLLOWER
07:40:46.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.857661 10783 catalog_manager.cc:1259] Loaded cluster ID: c328fd767aef487197fe7a5d3be0ea84
07:40:46.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.857942 10783 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e: loading cluster ID for follower catalog manager: success
07:40:46.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.865666 10783 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e: acquiring CA information for follower catalog manager: success
07:40:46.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:46.883523 10783 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 54cd480bcfd04419a4e71a3039c90a9e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:40:47.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:47.038580 10756 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:40:47.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:47.039626 10756 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:40:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:47.041214 10756 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:40:47.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:47.098040 10756 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:40:47.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:47.099764 10756 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.29.129
07:40:47.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.146174 10739 catalog_manager.cc:1259] Loaded cluster ID: c328fd767aef487197fe7a5d3be0ea84
07:40:47.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.146920 10739 catalog_manager.cc:1552] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: loading cluster ID for follower catalog manager: success
07:40:47.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.177870 10739 catalog_manager.cc:1574] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: acquiring CA information for follower catalog manager: success
07:40:47.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.187019 10739 catalog_manager.cc:1602] T 00000000000000000000000000000000 P dd0b61b1e14747358d6130a18fea1817: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:40:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.187935 10707 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "54cd480bcfd04419a4e71a3039c90a9e" member_type: VOTER last_known_addr { host: "127.10.29.190" port: 38881 } } peers { permanent_uuid: "dd0b61b1e14747358d6130a18fea1817" member_type: VOTER last_known_addr { host: "127.10.29.189" port: 45741 } } peers { permanent_uuid: "aa4b4ac2ec7a4b03a45d3c1a1b1a66a8" member_type: VOTER last_known_addr { host: "127.10.29.188" port: 43795 } } }
07:40:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.188807 10707 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa4b4ac2ec7a4b03a45d3c1a1b1a66a8 [sys.catalog]: This master's current role is: LEADER
07:40:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.236866 10756 tablet_server_runner.cc:78] Tablet server non-default flags:
07:40:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/data
07:40:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/wal
07:40:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.129:0
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/data/info.pb
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.129
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.29.129
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/logs
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:40:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:40:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:40:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:40:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:40:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:40:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:40:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:40:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:40:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.245658 10756 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:40:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:47.248363 10756 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:40:47.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:47.300431 10797 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:47.305084 10798 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:49.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:47.307969 10800 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:49.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:49.773993 10799 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2473 milliseconds
07:40:49.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.774122 10756 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:40:49.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.776206 10756 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:40:49.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.779242 10756 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:40:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.780946 10756 hybrid_clock.cc:648] HybridClock initialized: now 1786002049780911 us; error 58 us; skew 500 ppm
07:40:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.781816 10756 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:40:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.794332 10756 webserver.cc:460] Webserver started at http://127.10.29.129:45231/ using document root <none> and password file <none>
07:40:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.795331 10756 fs_manager.cc:359] Metadata directory not provided
07:40:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.795574 10756 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:40:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.796141 10756 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:40:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.801240 10756 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/data/instance:
07:40:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70ee5b381b6e4167970f150fa5ce26ff"
07:40:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:49 on dist-test-slave-04bb"
07:40:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.802614 10756 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/wal/instance:
07:40:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70ee5b381b6e4167970f150fa5ce26ff"
07:40:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:49 on dist-test-slave-04bb"
07:40:49.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.811825 10756 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
07:40:49.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.819320 10809 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:40:49.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.820600 10756 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
07:40:49.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.820996 10756 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/wal
07:40:49.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70ee5b381b6e4167970f150fa5ce26ff"
07:40:49.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:49 on dist-test-slave-04bb"
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.821413 10756 fs_report.cc:383] FS layout report
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/wal
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/wal
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/data/data
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:40:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:40:49.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.875782 10756 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:40:49.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.877591 10756 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:40:49.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.878091 10756 kserver.cc:163] Server-wide thread pool size limit: 3276
07:40:49.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.880937 10756 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:40:49.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.886440 10756 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:40:49.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.886725 10756 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:40:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.887066 10756 ts_tablet_manager.cc:610] Registered 0 tablets
07:40:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:49.887279 10756 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:40:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.051812 10756 rpc_server.cc:307] RPC server started. Bound to: 127.10.29.129:32941
07:40:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.051923 10975 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.29.129:32941 every 8 connection(s)
07:40:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.054539 10756 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/data/info.pb
07:40:50.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.065353 10358 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 10756
07:40:50.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.066212 10358 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-0/wal/instance
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.098933 10358 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/wal
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/data
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/logs
07:40:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/data/info.pb
07:40:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:40:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:40:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:40:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:40:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.130:0
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.29.130
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.130
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:40:50.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.113497 10977 heartbeater.cc:347] Connected to a master server at 127.10.29.189:45741
07:40:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.113735 10976 heartbeater.cc:347] Connected to a master server at 127.10.29.188:43795
07:40:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.114037 10977 heartbeater.cc:464] Registering TS with master...
07:40:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.114400 10976 heartbeater.cc:464] Registering TS with master...
07:40:50.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.115377 10977 heartbeater.cc:511] Master 127.10.29.189:45741 requested a full tablet report, sending...
07:40:50.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.115491 10976 heartbeater.cc:511] Master 127.10.29.188:43795 requested a full tablet report, sending...
07:40:50.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.116367 10979 heartbeater.cc:347] Connected to a master server at 127.10.29.190:38881
07:40:50.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.116727 10979 heartbeater.cc:464] Registering TS with master...
07:40:50.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.117702 10979 heartbeater.cc:511] Master 127.10.29.190:38881 requested a full tablet report, sending...
07:40:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.121300 10636 ts_manager.cc:194] Registered new tserver with Master: 70ee5b381b6e4167970f150fa5ce26ff (127.10.29.129:32941)
07:40:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.121955 10526 ts_manager.cc:194] Registered new tserver with Master: 70ee5b381b6e4167970f150fa5ce26ff (127.10.29.129:32941)
07:40:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.123991 10636 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.29.129:59135
07:40:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.125120 10417 ts_manager.cc:194] Registered new tserver with Master: 70ee5b381b6e4167970f150fa5ce26ff (127.10.29.129:32941)
07:40:50.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:50.422065 10993 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:40:50.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:50.422643 10993 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:40:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:50.423301 10993 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:40:50.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:50.454524 10993 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:40:50.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:50.455490 10993 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.29.130
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.489681 10993 tablet_server_runner.cc:78] Tablet server non-default flags:
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/data
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/wal
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.130:0
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/data/info.pb
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.130
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:40:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.29.130
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/logs
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:40:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.490876 10993 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:40:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:50.492807 10993 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:40:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:50.513622 11003 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.127902 10976 heartbeater.cc:503] Master 127.10.29.188:43795 was elected leader, sending a full tablet report...
07:40:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:50.514082 11007 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:50.513727 11002 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:51.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:51.813040 11005 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1297 milliseconds
07:40:51.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.813148 10993 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:40:51.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.814563 10993 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:40:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.817391 10993 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:40:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.818907 10993 hybrid_clock.cc:648] HybridClock initialized: now 1786002051818842 us; error 77 us; skew 500 ppm
07:40:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.819752 10993 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:40:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.827026 10993 webserver.cc:460] Webserver started at http://127.10.29.130:44275/ using document root <none> and password file <none>
07:40:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.827999 10993 fs_manager.cc:359] Metadata directory not provided
07:40:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.828238 10993 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:40:51.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.829383 10993 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:40:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.834291 10993 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/data/instance:
07:40:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8dfcfc93ba44b529e63c8cd92209dac"
07:40:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:51 on dist-test-slave-04bb"
07:40:51.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.835539 10993 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/wal/instance:
07:40:51.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8dfcfc93ba44b529e63c8cd92209dac"
07:40:51.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:51 on dist-test-slave-04bb"
07:40:51.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.842746 10993 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
07:40:51.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.849534 11020 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:40:51.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.851001 10993 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
07:40:51.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.851408 10993 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/wal
07:40:51.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8dfcfc93ba44b529e63c8cd92209dac"
07:40:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:51 on dist-test-slave-04bb"
07:40:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.851866 10993 fs_report.cc:383] FS layout report
07:40:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:40:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/wal
07:40:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/wal
07:40:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/data/data
07:40:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:40:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:40:51.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.898911 10993 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:40:51.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.900332 10993 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:40:51.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.900815 10993 kserver.cc:163] Server-wide thread pool size limit: 3276
07:40:51.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.903494 10993 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:40:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.907799 10993 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:40:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.908033 10993 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:40:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.908252 10993 ts_tablet_manager.cc:610] Registered 0 tablets
07:40:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:51.908380 10993 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:40:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.059293 10993 rpc_server.cc:307] RPC server started. Bound to: 127.10.29.130:42623
07:40:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.059465 11179 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.29.130:42623 every 8 connection(s)
07:40:52.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.062142 10993 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/data/info.pb
07:40:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.071977 10358 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 10993
07:40:52.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.072480 10358 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-1/wal/instance
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.110927 10358 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/wal
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/data
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/logs
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/data/info.pb
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.131:0
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.29.131
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.131
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:40:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.121152 11180 heartbeater.cc:347] Connected to a master server at 127.10.29.188:43795
07:40:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.121667 11180 heartbeater.cc:464] Registering TS with master...
07:40:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.122406 11181 heartbeater.cc:347] Connected to a master server at 127.10.29.189:45741
07:40:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.122769 11181 heartbeater.cc:464] Registering TS with master...
07:40:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.123005 11180 heartbeater.cc:511] Master 127.10.29.188:43795 requested a full tablet report, sending...
07:40:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.123803 11181 heartbeater.cc:511] Master 127.10.29.189:45741 requested a full tablet report, sending...
07:40:52.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.126129 10636 ts_manager.cc:194] Registered new tserver with Master: c8dfcfc93ba44b529e63c8cd92209dac (127.10.29.130:42623)
07:40:52.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.126492 10526 ts_manager.cc:194] Registered new tserver with Master: c8dfcfc93ba44b529e63c8cd92209dac (127.10.29.130:42623)
07:40:52.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.128172 10636 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.29.130:58959
07:40:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.137384 11182 heartbeater.cc:347] Connected to a master server at 127.10.29.190:38881
07:40:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.137769 11182 heartbeater.cc:464] Registering TS with master...
07:40:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.138494 11182 heartbeater.cc:511] Master 127.10.29.190:38881 requested a full tablet report, sending...
07:40:52.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.141024 10417 ts_manager.cc:194] Registered new tserver with Master: c8dfcfc93ba44b529e63c8cd92209dac (127.10.29.130:42623)
07:40:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:52.444707 11190 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:40:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:52.445276 11190 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:40:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:52.445971 11190 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:40:52.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:52.477689 11190 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:40:52.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:52.478739 11190 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.29.131
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.513888 11190 tablet_server_runner.cc:78] Tablet server non-default flags:
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.29.148:40139
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/data
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/wal
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.29.131:0
07:40:52.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/data/info.pb
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.29.131
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.29.190:38881,127.10.29.189:45741,127.10.29.188:43795
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.29.131
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/logs
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:40:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.515542 11190 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:40:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:52.517594 11190 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:40:52.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:52.538925 11203 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.132539 11180 heartbeater.cc:503] Master 127.10.29.188:43795 was elected leader, sending a full tablet report...
07:40:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:52.541477 11202 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:53.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:52.539790 11207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:53.843035 11206 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1299 milliseconds
07:40:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.843178 11190 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:40:53.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.844724 11190 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:40:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.847400 11190 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:40:53.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.848770 11190 hybrid_clock.cc:648] HybridClock initialized: now 1786002053848720 us; error 53 us; skew 500 ppm
07:40:53.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.849713 11190 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:40:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.857100 11190 webserver.cc:460] Webserver started at http://127.10.29.131:44549/ using document root <none> and password file <none>
07:40:53.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.858013 11190 fs_manager.cc:359] Metadata directory not provided
07:40:53.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.858210 11190 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:40:53.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.858606 11190 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:40:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.863735 11190 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/data/instance:
07:40:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "793d7d6e4a664020b4f2732609e77bf5"
07:40:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:53 on dist-test-slave-04bb"
07:40:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.864885 11190 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/wal/instance:
07:40:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "793d7d6e4a664020b4f2732609e77bf5"
07:40:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:53 on dist-test-slave-04bb"
07:40:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.872082 11190 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
07:40:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.878870 11220 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:40:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.880044 11190 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
07:40:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.880330 11190 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/wal
07:40:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "793d7d6e4a664020b4f2732609e77bf5"
07:40:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:53 on dist-test-slave-04bb"
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.880713 11190 fs_report.cc:383] FS layout report
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/wal
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/wal
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/data/data
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:40:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:40:53.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.922781 11190 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:40:53.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.924218 11190 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:40:53.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.924717 11190 kserver.cc:163] Server-wide thread pool size limit: 3276
07:40:53.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.927418 11190 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:40:53.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.931792 11190 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:40:53.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.931993 11190 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:40:53.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.932252 11190 ts_tablet_manager.cc:610] Registered 0 tablets
07:40:53.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:53.932420 11190 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:40:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.091748 11190 rpc_server.cc:307] RPC server started. Bound to: 127.10.29.131:40621
07:40:54.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.091884 11372 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.29.131:40621 every 8 connection(s)
07:40:54.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.094794 11190 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/data/info.pb
07:40:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.096024 10358 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 11190
07:40:54.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.096726 10358 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster1921888453719950973/ts-2/wal/instance
07:40:54.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.170723 11373 heartbeater.cc:347] Connected to a master server at 127.10.29.188:43795
07:40:54.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.171290 11373 heartbeater.cc:464] Registering TS with master...
07:40:54.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.171502 11374 heartbeater.cc:347] Connected to a master server at 127.10.29.189:45741
07:40:54.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.171927 11374 heartbeater.cc:464] Registering TS with master...
07:40:54.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.172281 11377 heartbeater.cc:347] Connected to a master server at 127.10.29.190:38881
07:40:54.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.172623 11377 heartbeater.cc:464] Registering TS with master...
07:40:54.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.172942 11373 heartbeater.cc:511] Master 127.10.29.188:43795 requested a full tablet report, sending...
07:40:54.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.173163 11374 heartbeater.cc:511] Master 127.10.29.189:45741 requested a full tablet report, sending...
07:40:54.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.173696 11377 heartbeater.cc:511] Master 127.10.29.190:38881 requested a full tablet report, sending...
07:40:54.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.176378 10526 ts_manager.cc:194] Registered new tserver with Master: 793d7d6e4a664020b4f2732609e77bf5 (127.10.29.131:40621)
07:40:54.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.176478 10635 ts_manager.cc:194] Registered new tserver with Master: 793d7d6e4a664020b4f2732609e77bf5 (127.10.29.131:40621)
07:40:54.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.177430 10416 ts_manager.cc:194] Registered new tserver with Master: 793d7d6e4a664020b4f2732609e77bf5 (127.10.29.131:40621)
07:40:54.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.179913 10635 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.29.131:45735
07:40:54.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.188045 10358 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:40:54.201 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:40:54.203 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:40:54.213 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.29.190"
port: 38881
}
bound_http_address {
host: "127.10.29.190"
port: 36791
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.29.189"
port: 45741
}
bound_http_address {
host: "127.10.29.189"
port: 45007
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.29.188"
port: 43795
}
bound_http_address {
host: "127.10.29.188"
port: 43195
}
}
}
07:40:54.218 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:40:54.220 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.29.129"
port: 32941
}
bound_http_address {
host: "127.10.29.129"
port: 45231
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.29.130"
port: 42623
}
bound_http_address {
host: "127.10.29.130"
port: 44275
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.29.131"
port: 40621
}
bound_http_address {
host: "127.10.29.131"
port: 44549
}
}
}
07:40:54.220 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:40:54.395 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:40:54.515 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskzm7lNz/test-tmp
07:40:54.517 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:40:54.517 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskzm7lNz/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:40:54.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:40:54.911 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:40:54.911 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:40:54.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.914624 11385 mini_oidc.cc:152] Starting JWKS server
07:40:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.918594 11385 webserver.cc:460] Webserver started at http://127.0.0.1:33087/ using document root <none> and password file <none>
07:40:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.918991 11385 mini_oidc.cc:189] Starting OIDC Discovery server
07:40:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:54.920989 11385 webserver.cc:460] Webserver started at http://127.0.0.1:40345/ using document root <none> and password file <none>
07:40:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:40:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/principal' for realm 'KRBTEST.COM',
07:40:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:40:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:40:55 dist-test-slave-04bb krb5kdc[11403](info): setting up network...
07:40:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:40:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:40:55 dist-test-slave-04bb krb5kdc[11403](info): set up 2 sockets
07:40:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:40:55 dist-test-slave-04bb krb5kdc[11403](info): commencing operation
07:40:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:40:55.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:55.184300 11373 heartbeater.cc:503] Master 127.10.29.188:43795 was elected leader, sending a full tablet report...
07:40:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.298728 11385 mini_kdc.cc:121] Time spent starting KDC: real 1.352s user 0.001s sys 0.021s
07:40:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:40:56.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:40:56.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:40:56.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:40:56.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:40:56.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:40:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:40:56.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:40:56.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:40:56.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:40:56.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/test-user.keytab.
07:40:56.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/test-user.keytab.
07:40:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:40:56 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002056, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:40:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:40:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:40:56Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:40:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:40:56Z Disabled control of system clock
07:40:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.30.126@KRBTEST.COM; defaulting to no policy
07:40:56.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:40:56.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.30.126@KRBTEST.COM" created.
07:40:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:40:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.30.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.126.keytab.
07:40:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.30.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.126.keytab.
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:56.501258 11385 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/logs
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/info.pb
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.126:33599
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.126
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.126.keytab
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.126
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:40:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:40:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:40:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:40:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:40:56.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.854362 11420 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:40:56.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.854900 11420 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:40:56.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.855466 11420 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:40:56.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.892319 11420 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:40:56.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.892603 11420 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:40:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.892889 11420 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:40:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.893082 11420 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:40:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.893273 11420 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:40:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.893544 11420 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:40:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.893781 11420 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33087/jwks.json
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:56.935966 11420 master_runner.cc:386] Master server non-default flags:
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.126.keytab
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.126
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.126:33599
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/info.pb
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.126
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/logs
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:56.937170 11420 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:40:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:56.938114 11420 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:40:56.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.958763 11430 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:58.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.958946 11433 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:58.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:56.958784 11429 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:58.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.197050 11431 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:40:58.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.197182 11420 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:40:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.201651 11420 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:40:58.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.204989 11420 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:40:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.206511 11420 hybrid_clock.cc:648] HybridClock initialized: now 1786002058206460 us; error 125 us; skew 500 ppm
07:40:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.207412 11420 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:40:58.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:40:58 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002058, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:40:58.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.214339 11420 init.cc:377] Logged in from keytab as kudu/127.11.30.126@KRBTEST.COM (short username kudu)
07:40:58.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.220264 11420 webserver.cc:460] Webserver started at http://127.11.30.126:41419/ using document root <none> and password file <none>
07:40:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.221282 11420 fs_manager.cc:359] Metadata directory not provided
07:40:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.221525 11420 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:40:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.221962 11420 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:40:58.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.227365 11420 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/instance:
07:40:58.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34da09c3e11249a69aafaf6209b7776c"
07:40:58.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:58 on dist-test-slave-04bb"
07:40:58.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.228641 11420 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal/instance:
07:40:58.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34da09c3e11249a69aafaf6209b7776c"
07:40:58.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:58 on dist-test-slave-04bb"
07:40:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.236877 11420 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
07:40:58.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.242584 11446 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:40:58.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.243966 11420 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
07:40:58.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.244310 11420 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:40:58.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34da09c3e11249a69aafaf6209b7776c"
07:40:58.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:58 on dist-test-slave-04bb"
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.244676 11420 fs_report.cc:383] FS layout report
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/data
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:40:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:40:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.287963 11420 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:40:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.293615 11420 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:40:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.294027 11420 kserver.cc:163] Server-wide thread pool size limit: 3276
07:40:58.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.374590 11420 rpc_server.cc:307] RPC server started. Bound to: 127.11.30.126:33599
07:40:58.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.374688 11517 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.30.126:33599 every 8 connection(s)
07:40:58.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.377430 11420 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/info.pb
07:40:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.379695 11385 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 11420
07:40:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.380306 11385 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal/instance
07:40:58.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.383565 11518 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:40:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.30.125@KRBTEST.COM; defaulting to no policy
07:40:58.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.397974 11518 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } has no permanent_uuid. Determining permanent_uuid...
07:40:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:40:58.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.30.125@KRBTEST.COM" created.
07:40:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:40:58 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002058, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.126@KRBTEST.COM for kudu/127.11.30.126@KRBTEST.COM
07:40:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:40:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.30.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.125.keytab.
07:40:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.30.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.125.keytab.
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.432672 11385 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/logs
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/info.pb
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.125:36485
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.125
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.125.keytab
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.125
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:40:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.442579 11518 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } has no permanent_uuid. Determining permanent_uuid...
07:40:58.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.444917 11450 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:40:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.449081 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.125:36485: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:40:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.498636 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } attempt: 1
07:40:58.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.502800 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.125:36485: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:40:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.571200 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } attempt: 2
07:40:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.575110 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.125:36485: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:40:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.666535 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } attempt: 3
07:40:58.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.670722 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.125:36485: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:40:58.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.752322 11525 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:40:58.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.753046 11525 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:40:58.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.753774 11525 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:40:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.812597 11525 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:40:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.812894 11525 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:40:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.813356 11525 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:40:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.813699 11525 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:40:58.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.814198 11525 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:40:58.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.814162 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } attempt: 4
07:40:58.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.814734 11525 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:40:58.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.815070 11525 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33087/jwks.json
07:40:58.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.824856 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.125:36485: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.865897 11525 master_runner.cc:386] Master server non-default flags:
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.125.keytab
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.125
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.125:36485
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:40:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/info.pb
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.125
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:40:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/logs
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.867218 11525 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:40:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.868264 11525 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:40:58.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.886425 11540 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:40:59.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:59.124305 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } attempt: 5
07:40:59.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:59.129506 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.125:36485: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:40:59.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:59.676901 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } attempt: 6
07:40:59.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:59.684474 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.125:36485: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:40:59.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:59.787439 11369 debug-util.cc:398] Leaking SignalData structure 0x7b08000af440 after lost signal to thread 11195
07:40:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:59.788319 11369 debug-util.cc:398] Leaking SignalData structure 0x7b08000af720 after lost signal to thread 11372
07:41:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.886935 11538 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:40:58.886415 11543 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:40:58.889588 11525 server_base.cc:1034] running on GCE node
07:41:00.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.147047 11525 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:41:00.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.150436 11525 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:41:00.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.151875 11525 hybrid_clock.cc:648] HybridClock initialized: now 1786002060151813 us; error 79 us; skew 500 ppm
07:41:00.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.152752 11525 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:41:00.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:00 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002060, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:00.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.164372 11525 init.cc:377] Logged in from keytab as kudu/127.11.30.125@KRBTEST.COM (short username kudu)
07:41:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.171856 11525 webserver.cc:460] Webserver started at http://127.11.30.125:39107/ using document root <none> and password file <none>
07:41:00.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.173290 11525 fs_manager.cc:359] Metadata directory not provided
07:41:00.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.173715 11525 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:41:00.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.174428 11525 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:41:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.180469 11525 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/instance:
07:41:00.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a99c610f30374504b8c907e07cc4c8bf"
07:41:00.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:41:00 on dist-test-slave-04bb"
07:41:00.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.181588 11525 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal/instance:
07:41:00.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a99c610f30374504b8c907e07cc4c8bf"
07:41:00.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:41:00 on dist-test-slave-04bb"
07:41:00.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.189141 11525 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
07:41:00.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.195322 11560 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:41:00.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.196437 11525 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
07:41:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.197459 11525 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:41:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a99c610f30374504b8c907e07cc4c8bf"
07:41:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:41:00 on dist-test-slave-04bb"
07:41:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.198186 11525 fs_report.cc:383] FS layout report
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/data
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:41:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:41:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.259258 11525 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:41:00.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.265839 11525 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:41:00.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.266433 11525 kserver.cc:163] Server-wide thread pool size limit: 3276
07:41:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.367069 11525 rpc_server.cc:307] RPC server started. Bound to: 127.11.30.125:36485
07:41:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.367226 11635 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.30.125:36485 every 8 connection(s)
07:41:00.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.369884 11525 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/info.pb
07:41:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.375990 11385 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 11525
07:41:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.376482 11385 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal/instance
07:41:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.377398 11636 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:41:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.396606 11636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } has no permanent_uuid. Determining permanent_uuid...
07:41:00.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.30.124@KRBTEST.COM; defaulting to no policy
07:41:00.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:41:00.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.30.124@KRBTEST.COM" created.
07:41:00.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:00 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002060, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.125@KRBTEST.COM for kudu/127.11.30.126@KRBTEST.COM
07:41:00.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:41:00.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.30.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.124.keytab.
07:41:00.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.30.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.124.keytab.
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.450047 11385 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/logs
07:41:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/info.pb
07:41:00.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:41:00.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:41:00.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:41:00.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.124:43367
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.124
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.124.keytab
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.124
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:41:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:41:00.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.458668 11636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } has no permanent_uuid. Determining permanent_uuid...
07:41:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:00 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002060, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.125@KRBTEST.COM for kudu/127.11.30.125@KRBTEST.COM
07:41:00.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.494386 11636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } has no permanent_uuid. Determining permanent_uuid...
07:41:00.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.497570 11564 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.511451 11636 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:00.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.560987 11636 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 1
07:41:00.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.566955 11636 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:00.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.635399 11636 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 2
07:41:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.639751 11636 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:00.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.731208 11636 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 3
07:41:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.735612 11636 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:00.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.744940 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } attempt: 7
07:41:00.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:00 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002058, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.126@KRBTEST.COM for kudu/127.11.30.125@KRBTEST.COM
07:41:00.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.777254 11518 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } has no permanent_uuid. Determining permanent_uuid...
07:41:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.783056 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:00.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.810976 11647 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:41:00.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.811678 11647 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:41:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.812433 11647 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:41:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.841511 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 1
07:41:00.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.847147 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.863086 11647 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:41:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.863384 11647 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:41:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.863744 11647 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:41:00.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.863951 11647 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:41:00.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.864136 11647 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:41:00.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.864333 11647 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:41:00.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.864470 11647 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33087/jwks.json
07:41:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.879104 11636 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 4
07:41:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.883582 11636 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.904551 11647 master_runner.cc:386] Master server non-default flags:
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.124.keytab
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.124
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.124:43367
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/info.pb
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.124
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:41:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/logs
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:41:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.906088 11647 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:41:00.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.907220 11647 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:41:00.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.928673 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 2
07:41:00.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.929265 11664 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:00.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.933867 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:01.019342 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 3
07:41:01.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:01.024011 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:01.164448 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 4
07:41:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:01.168640 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:01.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:01.183029 11636 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 5
07:41:01.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:01.186695 11636 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:01.452020 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 5
07:41:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:01.457732 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:01.734081 11636 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 6
07:41:01.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:01.744675 11636 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:02.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.010438 11518 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 6
07:41:02.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:02.017855 11518 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.30.124:43367: Network error: Client connection negotiation failed: client connection to 127.11.30.124:43367: connect: Connection refused (error 111)
07:41:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.932703 11667 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:00.935313 11647 server_base.cc:1034] running on GCE node
07:41:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:00.930262 11663 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.304316 11647 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:41:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.307546 11647 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:41:02.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.308984 11647 hybrid_clock.cc:648] HybridClock initialized: now 1786002062308939 us; error 62 us; skew 500 ppm
07:41:02.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.309814 11647 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:41:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002062, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.316048 11647 init.cc:377] Logged in from keytab as kudu/127.11.30.124@KRBTEST.COM (short username kudu)
07:41:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.321830 11647 webserver.cc:460] Webserver started at http://127.11.30.124:40497/ using document root <none> and password file <none>
07:41:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.322851 11647 fs_manager.cc:359] Metadata directory not provided
07:41:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.323041 11647 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:41:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.323417 11647 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:41:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.329376 11647 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/instance:
07:41:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73d6023b028c48fb96b5f7825af2ebcc"
07:41:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:41:02 on dist-test-slave-04bb"
07:41:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.330667 11647 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal/instance:
07:41:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73d6023b028c48fb96b5f7825af2ebcc"
07:41:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:41:02 on dist-test-slave-04bb"
07:41:02.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.338817 11647 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
07:41:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.345265 11690 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:41:02.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.346467 11647 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:41:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.346909 11647 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73d6023b028c48fb96b5f7825af2ebcc"
07:41:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:41:02 on dist-test-slave-04bb"
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.347296 11647 fs_report.cc:383] FS layout report
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/data
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:41:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:41:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.395882 11647 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:41:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.401938 11647 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:41:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.402524 11647 kserver.cc:163] Server-wide thread pool size limit: 3276
07:41:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.481352 11647 rpc_server.cc:307] RPC server started. Bound to: 127.11.30.124:43367
07:41:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.481451 11766 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.30.124:43367 every 8 connection(s)
07:41:02.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.483971 11647 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/info.pb
07:41:02.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.489672 11385 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 11647
07:41:02.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.490135 11385 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal/instance
07:41:02.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.490778 11767 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:41:02.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.505587 11767 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } has no permanent_uuid. Determining permanent_uuid...
07:41:02.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002056, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.30.124@KRBTEST.COM
07:41:02.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002062, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.124@KRBTEST.COM for kudu/127.11.30.126@KRBTEST.COM
07:41:02.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.555074 11767 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } has no permanent_uuid. Determining permanent_uuid...
07:41:02.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002056, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.30.125@KRBTEST.COM
07:41:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002062, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.124@KRBTEST.COM for kudu/127.11.30.125@KRBTEST.COM
07:41:02.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.580626 11767 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } has no permanent_uuid. Determining permanent_uuid...
07:41:02.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002056, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.30.126@KRBTEST.COM
07:41:02.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002062, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.124@KRBTEST.COM for kudu/127.11.30.124@KRBTEST.COM
07:41:02.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.605070 11385 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:41:02.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.610047 11767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: Bootstrap starting.
07:41:02.615 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:02.617 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:41:02.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.618702 11767 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: Neither blocks nor log segments found. Creating new log.
07:41:02.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.621521 11767 log.cc:824] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: Log is configured to *not* fsync() on all Append() calls
07:41:02.630 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:41:02.631 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:41:02.634 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.30.126"
port: 33599
}
bound_http_address {
host: "127.11.30.126"
port: 41419
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.30.125"
port: 36485
}
bound_http_address {
host: "127.11.30.125"
port: 39107
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.30.124"
port: 43367
}
bound_http_address {
host: "127.11.30.124"
port: 40497
}
}
}
07:41:02.634 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:41:02.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:02.635 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:41:02.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:41:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.644390 11767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: No bootstrap required, opened a new log
07:41:02.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002062, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:02.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:41:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.678948 11767 raft_consensus.cc:348] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.679749 11767 raft_consensus.cc:374] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:41:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.680027 11767 raft_consensus.cc:729] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73d6023b028c48fb96b5f7825af2ebcc, State: Initialized, Role: FOLLOWER
07:41:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.680977 11767 consensus_queue.cc:260] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:02.688 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:02.690 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc
07:41:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.692760 11767 sys_catalog.cc:564] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [sys.catalog]: configured and running, proceeding with master startup.
07:41:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.693480 11782 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } } }
07:41:02.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.694406 11782 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [sys.catalog]: This master's current role is: FOLLOWER
07:41:02.701 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:41:02.703 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc
07:41:02.708 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:41:02.713 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:41:02.716 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:41:02.717 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:41:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:02.756623 11802 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:02.756985 11802 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:02 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002062, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:41:02.774 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:02.774 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc
07:41:02.776 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:41:02.776 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.30.124:43367
07:41:02.778 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:41:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.779649 11385 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 11647
07:41:02.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.805091 11636 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } attempt: 7
07:41:02.809 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:02.809 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.30.125:36485
07:41:02.809 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:41:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.810133 11385 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 11525
07:41:02.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:02.837 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.30.126:33599
07:41:02.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:41:02.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.838274 11385 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 11420
07:41:02.863 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:02.863 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.30.124:43367
07:41:02.865 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:02.866971 11385 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/logs
07:41:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/info.pb
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.124:43367
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.124
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40497
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.124.keytab
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.124
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:41:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:41:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:41:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.195729 11808 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:41:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.196312 11808 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:41:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.197002 11808 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:41:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.232600 11808 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:41:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.232848 11808 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:41:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.233129 11808 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:41:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.233330 11808 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:41:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.233520 11808 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:41:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.233714 11808 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:41:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.233850 11808 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33087/jwks.json
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:03.273865 11808 master_runner.cc:386] Master server non-default flags:
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.124.keytab
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.124
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.124:43367
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/info.pb
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.124
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40497
07:41:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/logs
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:41:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:03.275024 11808 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:41:03.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:03.276105 11808 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:41:03.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.293267 11821 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:04.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.293654 11818 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:04.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:03.296278 11815 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:04.575820 11820 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:41:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.575884 11808 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:41:04.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.579955 11808 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:41:04.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.582540 11808 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:41:04.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.583923 11808 hybrid_clock.cc:648] HybridClock initialized: now 1786002064583884 us; error 40 us; skew 500 ppm
07:41:04.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.584774 11808 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:41:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:04 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002064, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.590729 11808 init.cc:377] Logged in from keytab as kudu/127.11.30.124@KRBTEST.COM (short username kudu)
07:41:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.595805 11808 webserver.cc:460] Webserver started at http://127.11.30.124:40497/ using document root <none> and password file <none>
07:41:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.596700 11808 fs_manager.cc:359] Metadata directory not provided
07:41:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.596897 11808 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:41:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.605057 11808 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
07:41:04.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.609411 11837 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:41:04.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.610486 11808 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.610780 11808 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73d6023b028c48fb96b5f7825af2ebcc"
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:41:02 on dist-test-slave-04bb"
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.613176 11808 fs_report.cc:383] FS layout report
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/wal
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/data
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:41:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:41:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:41:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:41:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:41:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.659875 11808 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:41:04.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.665899 11808 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:41:04.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.666424 11808 kserver.cc:163] Server-wide thread pool size limit: 3276
07:41:04.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.742167 11808 rpc_server.cc:307] RPC server started. Bound to: 127.11.30.124:43367
07:41:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.742275 11907 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.30.124:43367 every 8 connection(s)
07:41:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.745390 11808 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-2/data/info.pb
07:41:04.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.749857 11385 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 11808
07:41:04.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:04.750 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.30.125:36485
07:41:04.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.752727 11385 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/logs
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/info.pb
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:41:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.125:36485
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.125
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39107
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.125.keytab
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.125
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:41:04.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.759981 11909 sys_catalog.cc:263] Verifying existing consensus state
07:41:04.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.765537 11909 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: Bootstrap starting.
07:41:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.798666 11909 log.cc:824] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: Log is configured to *not* fsync() on all Append() calls
07:41:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.804399 11909 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:41:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.805444 11909 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: Bootstrap complete.
07:41:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.843655 11909 raft_consensus.cc:348] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.847198 11909 raft_consensus.cc:374] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:41:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.847507 11909 raft_consensus.cc:729] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73d6023b028c48fb96b5f7825af2ebcc, State: Initialized, Role: FOLLOWER
07:41:04.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.848577 11909 consensus_queue.cc:260] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.851776 11915 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } } }
07:41:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.852448 11915 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [sys.catalog]: This master's current role is: FOLLOWER
07:41:04.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.853878 11909 sys_catalog.cc:564] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [sys.catalog]: configured and running, proceeding with master startup.
07:41:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:04.889618 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:04.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:04.900123 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:04.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:04.922011 11845 tablet.cc:2369] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:41:04.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.904006 11808 master_runner.cc:186] Error getting master registration for 127.11.30.126:33599: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111),
07:41:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:04.930922 11808 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:41:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:05.002343 11915 raft_consensus.cc:482] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:41:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:05.002954 11915 raft_consensus.cc:504] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:05.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:05.006767 11915 leader_election.cc:288] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599), a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485)
07:41:05.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.010253 11838 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111)
07:41:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.022871 11838 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599): Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111)
07:41:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.025406 11843 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485): Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:41:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:05.025756 11843 leader_election.cc:302] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 73d6023b028c48fb96b5f7825af2ebcc; no voters: 34da09c3e11249a69aafaf6209b7776c, a99c610f30374504b8c907e07cc4c8bf
07:41:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:05.026683 11915 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:41:05.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.166608 11913 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:41:05.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.167270 11913 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:41:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.168197 11913 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:41:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.206018 11913 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:41:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.206248 11913 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:41:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.206513 11913 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:41:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.206712 11913 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:41:05.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.206897 11913 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:41:05.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.207080 11913 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:41:05.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.207216 11913 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33087/jwks.json
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:05.249729 11913 master_runner.cc:386] Master server non-default flags:
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.125.keytab
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.125
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:41:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.125:36485
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:41:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/info.pb
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.125
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39107
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/logs
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:41:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:05.251535 11913 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:41:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:05.253007 11913 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:41:05.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.272001 11950 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.905009 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.905334 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.273280 11949 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:05.271991 11952 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:06.617753 11951 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:41:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.617796 11913 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:41:06.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.623107 11913 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:41:06.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.626423 11913 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:41:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.627921 11913 hybrid_clock.cc:648] HybridClock initialized: now 1786002066627865 us; error 44 us; skew 500 ppm
07:41:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.629143 11913 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:41:06.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:06 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002066, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.642853 11913 init.cc:377] Logged in from keytab as kudu/127.11.30.125@KRBTEST.COM (short username kudu)
07:41:06.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.651777 11913 webserver.cc:460] Webserver started at http://127.11.30.125:39107/ using document root <none> and password file <none>
07:41:06.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.653012 11913 fs_manager.cc:359] Metadata directory not provided
07:41:06.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.653298 11913 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:41:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.664381 11913 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.002s sys 0.005s
07:41:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.670416 11961 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:41:06.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.672817 11913 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
07:41:06.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.673347 11913 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:41:06.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a99c610f30374504b8c907e07cc4c8bf"
07:41:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:41:00 on dist-test-slave-04bb"
07:41:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.676852 11913 fs_report.cc:383] FS layout report
07:41:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:41:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:41:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal
07:41:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/data
07:41:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:41:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.764037 11913 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:41:06.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.771436 11913 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:41:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.771992 11913 kserver.cc:163] Server-wide thread pool size limit: 3276
07:41:06.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.861603 11913 rpc_server.cc:307] RPC server started. Bound to: 127.11.30.125:36485
07:41:06.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.861672 12038 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.30.125:36485 every 8 connection(s)
07:41:06.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.865447 11913 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/data/info.pb
07:41:06.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.873565 11385 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 11913
07:41:06.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:06.875 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.30.126:33599
07:41:06.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.877610 11385 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/logs
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/info.pb
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.126:33599
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.126
07:41:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41419
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.126.keytab
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.126
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.878681 12039 sys_catalog.cc:263] Verifying existing consensus state
07:41:06.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 07:41:06.879654 12039 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:41:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.880317 11913 master.cc:537] Master@127.11.30.125:36485 shutting down...
07:41:06.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:06.909317 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:06.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:06.909895 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:06.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:06.964628 11913 master.cc:552] Master@127.11.30.125:36485 shutdown complete.
07:41:06.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:41:07.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.220957 12041 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:41:07.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.221505 12041 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:41:07.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.222234 12041 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:41:07.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.249191 12048 raft_consensus.cc:482] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:41:07.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.249603 12048 raft_consensus.cc:504] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.251276 12048 leader_election.cc:288] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599), a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485)
07:41:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.258420 11838 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599): Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111)
07:41:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.259081 11843 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485): Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:41:07.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.259976 11843 leader_election.cc:302] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 73d6023b028c48fb96b5f7825af2ebcc; no voters: 34da09c3e11249a69aafaf6209b7776c, a99c610f30374504b8c907e07cc4c8bf
07:41:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.260715 12048 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:41:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.261332 12041 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:41:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.261641 12041 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:41:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.262130 12041 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:41:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.262480 12041 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:41:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.262809 12041 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:41:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.263145 12041 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:41:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.263394 12041 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33087/jwks.json
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.319203 12041 master_runner.cc:386] Master server non-default flags:
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.30.84:33773
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.30.126:33599,127.11.30.125:36485,127.11.30.124:43367
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/krb5kdc/kudu_127.11.30.126.keytab
07:41:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.30.126
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.30.126:33599
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33087/jwks.json
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/info.pb
07:41:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.30.126
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41419
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/logs
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:11:51 UTC on ccycloud.quasar-fkskct.root.comops.site
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569619
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.320336 12041 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:41:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.321388 12041 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:41:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.338706 12055 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:07.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.913000 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:07.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.913295 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:08.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.338716 12057 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:08.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:07.340543 12059 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:41:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:07.341079 12041 server_base.cc:1034] running on GCE node
07:41:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.656575 12041 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.662334 12041 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:41:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.663998 12041 hybrid_clock.cc:648] HybridClock initialized: now 1786002068663771 us; error 251 us; skew 500 ppm
07:41:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.666343 12041 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:41:08.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:08 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002068, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:08.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.680267 12041 init.cc:377] Logged in from keytab as kudu/127.11.30.126@KRBTEST.COM (short username kudu)
07:41:08.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.690778 12041 webserver.cc:460] Webserver started at http://127.11.30.126:41419/ using document root <none> and password file <none>
07:41:08.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.692826 12041 fs_manager.cc:359] Metadata directory not provided
07:41:08.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.693329 12041 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:41:08.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.712118 12041 fs_manager.cc:709] Time spent opening directory manager: real 0.011s user 0.005s sys 0.004s
07:41:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.721975 12074 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:41:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.723981 12041 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.005s sys 0.003s
07:41:08.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.724471 12041 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data,/tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:41:08.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34da09c3e11249a69aafaf6209b7776c"
07:41:08.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:40:58 on dist-test-slave-04bb"
07:41:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.728462 12041 fs_report.cc:383] FS layout report
07:41:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:41:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/data
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:41:08.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.811132 12041 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:41:08.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.823346 12041 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:41:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.824326 12041 kserver.cc:163] Server-wide thread pool size limit: 3276
07:41:08.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:08.919708 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:08.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:08.920199 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.953018 12041 rpc_server.cc:307] RPC server started. Bound to: 127.11.30.126:33599
07:41:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.953363 12146 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.30.126:33599 every 8 connection(s)
07:41:08.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.957993 12041 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/data/info.pb
07:41:08.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.965935 11385 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu as pid 12041
07:41:08.967 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:41:08.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.979940 12147 sys_catalog.cc:263] Verifying existing consensus state
07:41:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 07:41:08.981508 12147 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:41:08.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:08.982348 12041 master.cc:537] Master@127.11.30.126:33599 shutting down...
07:41:09.068 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.30.124:43367(127.11.30.124:43367)] connecting to peer
07:41:09.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:09.148788 12041 master.cc:552] Master@127.11.30.126:33599 shutdown complete.
07:41:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskzm7lNz/test-tmp/mini-kudu-cluster6766636453652293108/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:41:09.187 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.189 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.265 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.11.30.124:43367(127.11.30.124:43367)] Successfully connected to peer
07:41:09.267 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.320 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.337 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:41:09.345 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.11.30.124
07:41:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:09 dist-test-slave-04bb krb5kdc[11403](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002062, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.30.124@KRBTEST.COM
07:41:09.551 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x54d41ecb, L:/127.0.0.1:39638 - R:/127.11.30.124:43367] using SASL/GSSAPI
07:41:09.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:09.596 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.600 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.602 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.602 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.603 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:09.614 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.615 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.617 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.617 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.621 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:09.633 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.634 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.637 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.640 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.640 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:09.654 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.656 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.657 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.657 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.657 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.663 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.664 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.664 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.664 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:09.673 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.674 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.676 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.676 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.677 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.679 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.680 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:09.693 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.695 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.697 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.697 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.700 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:09.754 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.757 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.757 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:09.894050 12158 raft_consensus.cc:482] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:41:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:09.894482 12158 raft_consensus.cc:504] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:09.896127 12158 leader_election.cc:288] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599), a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485)
07:41:09.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:09.903081 11843 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485): Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:41:09.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:09.903604 11838 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599): Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111)
07:41:09.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:09.903962 11838 leader_election.cc:302] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 73d6023b028c48fb96b5f7825af2ebcc; no voters: 34da09c3e11249a69aafaf6209b7776c, a99c610f30374504b8c907e07cc4c8bf
07:41:09.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:09.904672 12158 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:41:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:09.923482 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:09.923733 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:09.933113 11808 master_runner.cc:186] Error getting master registration for 127.11.30.126:33599: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111),
07:41:09.934 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:09.933650 11808 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:41:09.935 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:09.936 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.936 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.937 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:09.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.940 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.940 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:09.940 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:10.054 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:10.055 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:10.059 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:10.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:10.060 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:10.060 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:10.060 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:10.060 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:10.062 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:10.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:10.926589 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:10.926891 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:10.994 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:10.996 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:10.996 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:10.996 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:10.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:10.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:10.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:10.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:10.999 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:11.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:11.930079 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:11.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:11.930344 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:12.214 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:12.215 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:12.217 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:12.217 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:12.217 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:12.217 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:12.217 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:12.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:12.219 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:12.933704 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:12.934019 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:13.051029 10564 debug-util.cc:398] Leaking SignalData structure 0x7b08000b26a0 after lost signal to thread 10483
07:41:13.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:13.052160 10564 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc3a0 after lost signal to thread 10569
07:41:13.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:13.939409 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:13.939893 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:14.572330 12174 raft_consensus.cc:482] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:41:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:14.573014 12174 raft_consensus.cc:504] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:14.575280 12174 leader_election.cc:288] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599), a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485)
07:41:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:14.577785 11838 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:41:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:14.581801 11838 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599): Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111)
07:41:14.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:14.582796 11843 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485): Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:41:14.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:14.583279 11843 leader_election.cc:302] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 73d6023b028c48fb96b5f7825af2ebcc; no voters: 34da09c3e11249a69aafaf6209b7776c, a99c610f30374504b8c907e07cc4c8bf
07:41:14.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:14.584147 12174 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:41:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:14 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002074, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:14.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:14.595031 11828 init.cc:295] Successfully reacquired a new kerberos TGT
07:41:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:14.936278 11808 master_runner.cc:186] Error getting master registration for 127.11.30.126:33599: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111),
07:41:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:14.936852 11808 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:41:14.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:14.943009 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:14.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:14.943231 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:15.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:15.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:15.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:15.117 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:15.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:15.117 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:15.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:15.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:15.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:15.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:15.946084 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:15.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:15.946342 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:16.949757 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:16.950022 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:17.952896 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:17.953146 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:18.334 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:18.335 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:18.337 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:18.337 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:18.337 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:18.337 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:18.337 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:18.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:18.339 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:18.956154 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:18.956410 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:19.353 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:19.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:19.356 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:19.356 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:19.357 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:19.357 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:19.357 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:19.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:19.358 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:19.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:19.939714 11808 master_runner.cc:186] Error getting master registration for 127.11.30.126:33599: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111),
07:41:19.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:19.940114 11808 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:41:19.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:19.959196 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:19.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:19.959478 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:20.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:20.315 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:20.316 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:20.317 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:20.317 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:20.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:20.320 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:20.320 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:20.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:20.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:20.654182 12192 raft_consensus.cc:482] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:41:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:20.654567 12192 raft_consensus.cc:504] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:20.656188 12192 leader_election.cc:288] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599), a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485)
07:41:20.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:20.658492 11843 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111) [suppressed 1 similar messages]
07:41:20.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:20.661341 11838 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599): Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111)
07:41:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:20.661970 11843 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485): Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:41:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:20.662304 11843 leader_election.cc:302] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 73d6023b028c48fb96b5f7825af2ebcc; no voters: 34da09c3e11249a69aafaf6209b7776c, a99c610f30374504b8c907e07cc4c8bf
07:41:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:20.662948 12192 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:41:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:20.962513 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:20.962775 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:21.965664 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:21.965942 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:22.969354 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:22.969607 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:23.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:23.093040 12199 raft_consensus.cc:482] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:41:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:23.093436 12199 raft_consensus.cc:504] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:23.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:23.095106 12199 leader_election.cc:288] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599), a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485)
07:41:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:23.099570 11838 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599): Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111)
07:41:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:23.100221 11843 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485): Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:41:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:23.100780 11843 leader_election.cc:302] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 73d6023b028c48fb96b5f7825af2ebcc; no voters: 34da09c3e11249a69aafaf6209b7776c, a99c610f30374504b8c907e07cc4c8bf
07:41:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:23.101508 12199 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:41:23.134 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:23.135 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:23.136 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:23.136 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:23.137 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:23.137 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:23.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:23.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:23.138 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:23.973366 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:23.973783 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:24.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:24.495 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:24.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:24.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:24.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:24.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:24.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:24.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:24.500 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:24 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002084, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:24.597718 11828 init.cc:295] Successfully reacquired a new kerberos TGT
07:41:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:24.943796 11808 master_runner.cc:186] Error getting master registration for 127.11.30.126:33599: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111),
07:41:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:24.944233 11808 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:41:24.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:24.976807 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:24.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:24.977083 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:25.979903 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:25.980163 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:26.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:26.499661 11369 debug-util.cc:398] Leaking SignalData structure 0x7b08000af160 after lost signal to thread 11195
07:41:26.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:26.500591 11369 debug-util.cc:398] Leaking SignalData structure 0x7b08000afba0 after lost signal to thread 11372
07:41:26.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:26.983204 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:26.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:26.983474 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:27.974 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:27.975 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:27.976 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:27.976 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:27.977 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:27.977 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:27.977 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:27.977 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:27.978 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:27.986516 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:27.986799 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:28.694 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:28.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:28.697 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:28.697 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:28.697 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:28.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:28.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:28.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:28.700 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:28.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:28.989935 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:28.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:28.990199 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:29.947336 11808 master_runner.cc:186] Error getting master registration for 127.11.30.126:33599: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111),
07:41:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:29.947774 11808 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:41:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:29.993714 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:29.994029 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:30.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:30.997253 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:30.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:30.997516 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:32.000368 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:32.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:32.000698 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:32.134 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:32.135 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:32.136 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:32.136 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:32.136 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:32.136 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:32.136 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:32.136 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:32.138 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:33.003659 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:33.003911 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:33.174 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:33.175 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:33.176 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:33.176 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:33.176 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:33.177 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:33.177 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:33.177 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:33.178 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:33.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:33.335446 12225 raft_consensus.cc:482] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:41:33.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:33.336149 12225 raft_consensus.cc:504] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34da09c3e11249a69aafaf6209b7776c" member_type: VOTER last_known_addr { host: "127.11.30.126" port: 33599 } } peers { permanent_uuid: "a99c610f30374504b8c907e07cc4c8bf" member_type: VOTER last_known_addr { host: "127.11.30.125" port: 36485 } } peers { permanent_uuid: "73d6023b028c48fb96b5f7825af2ebcc" member_type: VOTER last_known_addr { host: "127.11.30.124" port: 43367 } }
07:41:33.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:33.338267 12225 leader_election.cc:288] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599), a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485)
07:41:33.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:33.341012 11838 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111) [suppressed 3 similar messages]
07:41:33.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:33.344465 11843 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a99c610f30374504b8c907e07cc4c8bf (127.11.30.125:36485): Network error: Client connection negotiation failed: client connection to 127.11.30.125:36485: connect: Connection refused (error 111)
07:41:33.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:33.344956 11838 leader_election.cc:334] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34da09c3e11249a69aafaf6209b7776c (127.11.30.126:33599): Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111)
07:41:33.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:33.345327 11838 leader_election.cc:302] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 73d6023b028c48fb96b5f7825af2ebcc; no voters: 34da09c3e11249a69aafaf6209b7776c, a99c610f30374504b8c907e07cc4c8bf
07:41:33.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:33.346045 12225 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:41:34.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:34.007287 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:34.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:34.007576 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:41:34 dist-test-slave-04bb krb5kdc[11403](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786002094, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.30.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:41:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:34.600293 11828 init.cc:295] Successfully reacquired a new kerberos TGT
07:41:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:34.950941 11808 master_runner.cc:186] Error getting master registration for 127.11.30.126:33599: Network error: Client connection negotiation failed: client connection to 127.11.30.126:33599: connect: Connection refused (error 111),
07:41:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:34.951380 11808 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:41:35.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:35.010598 11936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 73d6023b028c48fb96b5f7825af2ebcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:41:35.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:41:35.010871 11936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:41:35.254 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:35.254 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:35.255 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:35.255 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:35.255 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:35.255 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:35.255 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:35.256 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:35.257 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:35.813 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.125:36485(127.11.30.125:36485)] connecting to peer
07:41:35.814 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.30.126:33599(127.11.30.126:33599)] connecting to peer
07:41:35.815 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:35.815 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.30.125:36485(127.11.30.125:36485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:35.815 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.125:36485: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485
07:41:35.816 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:35.816 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.126:33599(127.11.30.126:33599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:35.816 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.30.126:33599: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
07:41:35.818 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599; will retry
07:41:35.822 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26846), Traces: [0ms] refreshing cache from master, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [290ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [341ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [587ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [622ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [626ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [627ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [639ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [640ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [643ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [653ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [662ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [678ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [686ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [698ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [701ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [718ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [721ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [778ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [780ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [782ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [957ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [960ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [961ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [962ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1073ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [1075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1084ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2014ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [2017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [2018ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2019ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [2020ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [2020ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3234ms] refreshing cache from master, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [3236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [3238ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [3239ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3240ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [3241ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [6134ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [6138ms] too many traces: truncated at 100 traces, deferred=Deferred@101578995(state=PENDING, result=null, callback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631), errback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631)))
07:41:35.827 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.30.126:33599(127.11.30.126:33599)] disconnecting while in state TERMINATED
07:41:35.828 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.30.124:43367(127.11.30.124:43367)] disconnecting while in state READY
07:41:35.828 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.30.125:36485(127.11.30.125:36485)] disconnecting while in state TERMINATED
07:41:35.834 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:41:35.838 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:41:35.839 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.11.30.124:43367(127.11.30.124:43367)] handling channelInactive
07:41:35.844 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.30.124:43367(127.11.30.124:43367)] cleaning up while in state READY due to: connection closed
07:41:35.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:35.853329 11385 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 11808
07:41:37.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:41:37Z chronyd exiting
07:41:38.087 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:41:38.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:38.089073 10358 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 10756
07:41:38.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:38.123539 10358 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 10993
07:41:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:38.157227 10358 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 11190
07:41:38.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:38.194294 10358 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 10374
07:41:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:38.232777 10358 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 10478
07:41:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:41:38.270328 10358 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzm7lNz/build/tsan/bin/kudu with pid 10573
07:41:39.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:41:39Z chronyd exiting
07:41:40.475 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26846), Traces: [0ms] refreshing cache from master, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [290ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [341ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [587ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [622ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [626ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [627ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [639ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [640ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [643ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [653ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [662ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [678ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [686ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [698ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [701ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [718ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [721ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [778ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [780ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [782ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [957ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [960ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [961ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [962ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1073ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [1075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1084ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2014ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [2017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [2018ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2019ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [2020ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [2020ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3234ms] refreshing cache from master, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [3236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [3238ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [3239ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3240ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [3241ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [6134ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [6138ms] too many traces: truncated at 100 traces, deferred=Deferred@101578995(state=PENDING, result=null, callback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631), errback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.30.126:33599
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26846), Traces: [0ms] refreshing cache from master, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [290ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [341ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [587ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [622ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [626ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [627ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [639ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [640ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [643ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [653ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [662ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [678ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [686ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [698ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [701ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [718ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [721ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [778ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [780ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [782ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [957ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [960ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [961ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [962ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1073ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [1075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1084ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2014ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [2017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [2018ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2019ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [2020ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [2020ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3234ms] refreshing cache from master, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [3236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [3238ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [3239ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3240ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [3241ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [6134ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [6138ms] too many traces: truncated at 100 traces, deferred=Deferred@101578995(state=PENDING, result=null, callback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631), errback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.30.126:33599
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
07:41:40.913 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
07:41:40.914 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26846), Traces: [0ms] refreshing cache from master, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [290ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [341ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [587ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [622ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [626ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [627ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [639ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [640ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [643ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [653ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [662ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [678ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [686ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [698ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [701ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [718ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [721ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [778ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [780ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [782ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [957ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [960ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [961ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [962ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1073ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [1075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1084ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2014ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [2017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [2018ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2019ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [2020ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [2020ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3234ms] refreshing cache from master, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [3236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [3238ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [3239ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3240ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [3241ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [6134ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [6138ms] too many traces: truncated at 100 traces, deferred=Deferred@101578995(state=PENDING, result=null, callback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631), errback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.30.126:33599
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26846), Traces: [0ms] refreshing cache from master, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [290ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [341ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [587ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [622ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [626ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [627ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [639ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [640ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [643ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [653ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [658ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [662ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [678ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [686ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [698ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [701ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [718ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [721ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [778ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [780ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [782ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [957ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [960ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [961ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [962ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1073ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [1075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [1084ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2014ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [2017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [2018ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [2019ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [2020ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [2020ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3234ms] refreshing cache from master, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [3235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [3236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [3238ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.125:36485: Network error: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485, [3239ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.126:33599: Network error: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [3240ms] Sub RPC ConnectToMaster: received response from server master-127.11.30.124:43367: OK, [3241ms] delaying RPC due to: Service unavailable: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599, [6134ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.124:43367, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.125:36485, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.30.126:33599, [6138ms] too many traces: truncated at 100 traces, deferred=Deferred@101578995(state=PENDING, result=null, callback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631), errback=(continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209) -> (continuation of Deferred@1609670028 after retry RPC after error@624813967) -> (continuation of Deferred@704519827 after retry RPC after error@730197203) -> (continuation of Deferred@2026572242 after retry RPC after error@847288539) -> (continuation of Deferred@1800606366 after retry RPC after error@1874106451) -> (continuation of Deferred@1326553582 after retry RPC after error@2036434583) -> (continuation of Deferred@181779655 after retry RPC after error@1105779662) -> (continuation of Deferred@41284852 after retry RPC after error@2010484499) -> (continuation of Deferred@1354839722 after retry RPC after error@1466999255) -> (continuation of Deferred@2056207769 after retry RPC after error@570005708) -> (continuation of Deferred@2027195967 after retry RPC after error@720955115) -> (continuation of Deferred@689468888 after retry RPC after error@1154131919) -> (continuation of Deferred@414415903 after retry RPC after error@1972444764) -> (continuation of Deferred@525144963 after retry RPC after error@245906298) -> (continuation of Deferred@234417460 after retry RPC after error@1885761833) -> (continuation of Deferred@130069933 after retry RPC after error@1224490436) -> (continuation of Deferred@44562594 after retry RPC after error@1099640550) -> (continuation of Deferred@1130661543 after retry RPC after error@1784643858) -> (continuation of Deferred@1459619347 after retry RPC after error@1332014603) -> (continuation of Deferred@676468990 after retry RPC after error@1049840631)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.30.124:43367,127.11.30.125:36485,127.11.30.126:33599) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.125:36485(127.11.30.125:36485): Connection refused: /127.11.30.125:36485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.30.126:33599(127.11.30.126:33599): Connection refused: /127.11.30.126:33599
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.30.126:33599
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)