05:33:14.765 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
05:33:14.777 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskSNHo_2/test-tmp
05:33:14.780 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:33:14.783 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskSNHo_2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:33:15.001 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
05:33:15.176 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:33:15.177 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:33:15.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.181185 16945 mini_oidc.cc:152] Starting JWKS server
05:33:15.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.185273 16945 webserver.cc:460] Webserver started at http://127.0.0.1:35057/ using document root <none> and password file <none>
05:33:15.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.185734 16945 mini_oidc.cc:189] Starting OIDC Discovery server
05:33:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.187424 16945 webserver.cc:460] Webserver started at http://127.0.0.1:33143/ using document root <none> and password file <none>
05:33:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:33:15Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:33:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:33:15Z Disabled control of system clock
05:33:15.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.236728 16945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:15.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:15.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/wal
05:33:15.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/data
05:33:15.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/logs
05:33:15.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/data/info.pb
05:33:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.126:42451
05:33:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.126
05:33:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35057/jwks.json with env {}
05:33:15.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.532199 16957 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.532764 16957 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.533394 16957 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.564261 16957 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.564595 16957 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.564790 16957 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.564999 16957 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.565272 16957 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.565477 16957 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35057/jwks.json
05:33:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.601692 16957 master_runner.cc:386] Master server non-default flags:
05:33:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/data
05:33:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/wal
05:33:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.126:42451
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35057/jwks.json
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/data/info.pb
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.126
05:33:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/logs
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.603114 16957 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.605261 16957 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:15.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.623014 16966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.623004 16963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:15.624845 16957 server_base.cc:1034] running on GCE node
05:33:16.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:15.623056 16964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.822392 16957 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:16.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.825802 16957 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.827298 16957 hybrid_clock.cc:648] HybridClock initialized: now 1790314396827279 us; error 106 us; skew 500 ppm
05:33:16.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.828583 16957 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:16.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.839838 16957 webserver.cc:460] Webserver started at http://127.16.140.126:36387/ using document root <none> and password file <none>
05:33:16.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.840903 16957 fs_manager.cc:359] Metadata directory not provided
05:33:16.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.841102 16957 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:16.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.841480 16957 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:16.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.846367 16957 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/data/instance:
05:33:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f01dd8bbee7e4b808226bda4398c0581"
05:33:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:16 on dist-test-slave-w8hg"
05:33:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.847621 16957 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/wal/instance:
05:33:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f01dd8bbee7e4b808226bda4398c0581"
05:33:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:16 on dist-test-slave-w8hg"
05:33:16.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.855969 16957 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
05:33:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.862061 16973 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.863237 16957 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:33:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.863601 16957 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/wal
05:33:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f01dd8bbee7e4b808226bda4398c0581"
05:33:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:16 on dist-test-slave-w8hg"
05:33:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.863978 16957 fs_report.cc:383] FS layout report
05:33:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/wal
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/wal
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/data/data
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:16.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.908198 16957 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.909919 16957 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.910372 16957 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:16.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.984388 16957 rpc_server.cc:307] RPC server started. Bound to: 127.16.140.126:42451
05:33:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.984475 17024 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.140.126:42451 every 8 connection(s)
05:33:16.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.987232 16957 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/data/info.pb
05:33:16.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.992444 17025 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:33:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.993938 16945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 16957
05:33:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.994448 16945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-0/wal/instance
05:33:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:16.999677 16945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/wal
05:33:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/data
05:33:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/logs
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/data/info.pb
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.125:35207
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.125
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35057/jwks.json with env {}
05:33:17.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.009107 17025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } has no permanent_uuid. Determining permanent_uuid...
05:33:17.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.038367 17025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } has no permanent_uuid. Determining permanent_uuid...
05:33:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.040689 16977 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.140.125:35207: connect: Connection refused (error 111)
05:33:17.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.045493 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.125:35207: Network error: Client connection negotiation failed: client connection to 127.16.140.125:35207: connect: Connection refused (error 111)
05:33:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.095027 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } attempt: 1
05:33:17.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.099201 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.125:35207: Network error: Client connection negotiation failed: client connection to 127.16.140.125:35207: connect: Connection refused (error 111)
05:33:17.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.167708 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } attempt: 2
05:33:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.171823 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.125:35207: Network error: Client connection negotiation failed: client connection to 127.16.140.125:35207: connect: Connection refused (error 111)
05:33:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.263316 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } attempt: 3
05:33:17.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.267452 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.125:35207: Network error: Client connection negotiation failed: client connection to 127.16.140.125:35207: connect: Connection refused (error 111)
05:33:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.320191 17027 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.320910 17027 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.321595 17027 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.353796 17027 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.354224 17027 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.354561 17027 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:17.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.354895 17027 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:17.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.355257 17027 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:17.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.355538 17027 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35057/jwks.json
05:33:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.392046 17027 master_runner.cc:386] Master server non-default flags:
05:33:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/data
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/wal
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.125:35207
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35057/jwks.json
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/data/info.pb
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.125
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/logs
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.393671 17027 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:17.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.395978 17027 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.410830 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } attempt: 4
05:33:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.413622 17037 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.416471 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.125:35207: Network error: Client connection negotiation failed: client connection to 127.16.140.125:35207: connect: Connection refused (error 111)
05:33:17.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:17.716071 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } attempt: 5
05:33:17.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.723187 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.125:35207: Network error: Client connection negotiation failed: client connection to 127.16.140.125:35207: connect: Connection refused (error 111)
05:33:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.270773 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } attempt: 6
05:33:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:18.283713 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.125:35207: Network error: Client connection negotiation failed: client connection to 127.16.140.125:35207: connect: Connection refused (error 111)
05:33:18.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.418159 17039 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:18.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:17.414316 17036 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:18.661480 17038 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1245 milliseconds
05:33:18.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.661612 17027 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:33:18.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.663041 17027 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.666245 17027 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:18.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.667677 17027 hybrid_clock.cc:648] HybridClock initialized: now 1790314398667639 us; error 67 us; skew 500 ppm
05:33:18.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.668596 17027 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:18.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.675413 17027 webserver.cc:460] Webserver started at http://127.16.140.125:44053/ using document root <none> and password file <none>
05:33:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.676432 17027 fs_manager.cc:359] Metadata directory not provided
05:33:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.676625 17027 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.677374 17027 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:18.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.682353 17027 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/data/instance:
05:33:18.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93b85063fc1c405eb2ea9609bb91b01f"
05:33:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:18 on dist-test-slave-w8hg"
05:33:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.683449 17027 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/wal/instance:
05:33:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93b85063fc1c405eb2ea9609bb91b01f"
05:33:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:18 on dist-test-slave-w8hg"
05:33:18.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.690853 17027 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
05:33:18.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.696995 17048 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:18.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.698294 17027 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
05:33:18.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.698627 17027 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/wal
05:33:18.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93b85063fc1c405eb2ea9609bb91b01f"
05:33:18.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:18 on dist-test-slave-w8hg"
05:33:18.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.698966 17027 fs_report.cc:383] FS layout report
05:33:18.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:18.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/wal
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/wal
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/data/data
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:18.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:18.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.735700 17027 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:18.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.737246 17027 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:18.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.737663 17027 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.809135 17027 rpc_server.cc:307] RPC server started. Bound to: 127.16.140.125:35207
05:33:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.809208 17099 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.140.125:35207 every 8 connection(s)
05:33:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.811916 17027 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/data/info.pb
05:33:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.816347 16945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17027
05:33:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.816709 16945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-1/wal/instance
05:33:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.817739 17100 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:33:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.818988 16945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/wal
05:33:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/data
05:33:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/logs
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/data/info.pb
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.124:40427
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.124
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35057/jwks.json with env {}
05:33:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.835831 17100 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } has no permanent_uuid. Determining permanent_uuid...
05:33:18.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.862941 17100 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } has no permanent_uuid. Determining permanent_uuid...
05:33:18.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.881029 17100 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } has no permanent_uuid. Determining permanent_uuid...
05:33:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:18.883140 17052 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:18.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:18.887406 17100 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:18.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:18.936949 17100 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 1
05:33:18.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:18.941795 17100 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.010264 17100 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 2
05:33:19.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.014071 17100 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.105497 17100 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 3
05:33:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.109440 17100 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.153554 17102 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:19.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.154150 17102 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:19.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.154614 17102 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:19.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.187492 17102 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.187814 17102 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.188009 17102 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.188282 17102 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.188550 17102 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.188692 17102 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35057/jwks.json
05:33:19.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.225320 17102 master_runner.cc:386] Master server non-default flags:
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/data
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/wal
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.124:40427
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35057/jwks.json
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/data/info.pb
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.124
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/logs
05:33:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.226644 17102 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.228781 17102 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:19.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.250453 17115 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.252873 17100 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 4
05:33:19.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.256546 17100 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.344302 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } attempt: 7
05:33:19.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.358044 17025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } has no permanent_uuid. Determining permanent_uuid...
05:33:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.363811 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.422231 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 1
05:33:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.427042 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.508524 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 2
05:33:19.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.513345 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.556005 17100 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 5
05:33:19.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.561082 17100 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.598742 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 3
05:33:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.603356 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.743829 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 4
05:33:19.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.748461 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:20.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.031913 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 5
05:33:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:20.035641 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:20.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.108520 17100 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 6
05:33:20.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:20.116174 17100 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.250705 17112 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:19.250461 17113 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:19.252002 17102 server_base.cc:1034] running on GCE node
05:33:20.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.472333 17102 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:20.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.475410 17102 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:20.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.476827 17102 hybrid_clock.cc:648] HybridClock initialized: now 1790314400476795 us; error 51 us; skew 500 ppm
05:33:20.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.477632 17102 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:20.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.485721 17102 webserver.cc:460] Webserver started at http://127.16.140.124:40685/ using document root <none> and password file <none>
05:33:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.487057 17102 fs_manager.cc:359] Metadata directory not provided
05:33:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.487251 17102 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.487643 17102 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:20.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.492816 17102 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/data/instance:
05:33:20.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f47357fc8fc9450aae0a29add9dd7731"
05:33:20.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:20 on dist-test-slave-w8hg"
05:33:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.493907 17102 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/wal/instance:
05:33:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f47357fc8fc9450aae0a29add9dd7731"
05:33:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:20 on dist-test-slave-w8hg"
05:33:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.502128 17102 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
05:33:20.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.508167 17126 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:20.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.509272 17102 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
05:33:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.509654 17102 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/wal
05:33:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f47357fc8fc9450aae0a29add9dd7731"
05:33:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:20 on dist-test-slave-w8hg"
05:33:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.510049 17102 fs_report.cc:383] FS layout report
05:33:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/wal
05:33:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/wal
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/data/data
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:20.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.546650 17102 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:20.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.548491 17102 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:20.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.549031 17102 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:20.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.588251 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 6
05:33:20.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:20.598604 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.140.124:40427: Network error: Client connection negotiation failed: client connection to 127.16.140.124:40427: connect: Connection refused (error 111)
05:33:20.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.631081 17102 rpc_server.cc:307] RPC server started. Bound to: 127.16.140.124:40427
05:33:20.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.631150 17179 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.140.124:40427 every 8 connection(s)
05:33:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.634109 17102 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/data/info.pb
05:33:20.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.635675 16945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17102
05:33:20.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.636252 16945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/master-2/wal/instance
05:33:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.641153 17180 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:33:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.658162 17180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } has no permanent_uuid. Determining permanent_uuid...
05:33:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.686866 17180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } has no permanent_uuid. Determining permanent_uuid...
05:33:20.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.698206 17180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } has no permanent_uuid. Determining permanent_uuid...
05:33:20.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.715260 17180 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731: Bootstrap starting.
05:33:20.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.724320 17180 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731: Neither blocks nor log segments found. Creating new log.
05:33:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.726630 17180 log.cc:824] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731: Log is configured to *not* fsync() on all Append() calls
05:33:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.732579 17180 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731: No bootstrap required, opened a new log
05:33:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.754702 17180 raft_consensus.cc:348] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.755340 17180 raft_consensus.cc:374] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:33:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.755532 17180 raft_consensus.cc:729] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f47357fc8fc9450aae0a29add9dd7731, State: Initialized, Role: FOLLOWER
05:33:20.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.756222 17180 consensus_queue.cc:260] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:20.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.758787 17189 sys_catalog.cc:455] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:20.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.759426 17189 sys_catalog.cc:458] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: This master's current role is: FOLLOWER
05:33:20.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.760301 17180 sys_catalog.cc:564] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: configured and running, proceeding with master startup.
05:33:20.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:20.788321 17200 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:20.788688 17200 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:20.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:20.804327 17132 tablet.cc:2369] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:33:20.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.782529 17102 master_runner.cc:186] Error getting master registration for 127.16.140.126:42451: OK, instance_id { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" instance_seqno: 1790314396964027 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:33:20.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.817037 17102 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:33:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.954823 17189 raft_consensus.cc:482] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:33:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.955341 17189 raft_consensus.cc:504] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:20.957476 17189 leader_election.cc:288] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f01dd8bbee7e4b808226bda4398c0581 (127.16.140.126:42451), 93b85063fc1c405eb2ea9609bb91b01f (127.16.140.125:35207)
05:33:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.176528 17100 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 7
05:33:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.203445 17100 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f: Bootstrap starting.
05:33:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.213053 17100 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f: Neither blocks nor log segments found. Creating new log.
05:33:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.215318 17100 log.cc:824] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f: Log is configured to *not* fsync() on all Append() calls
05:33:21.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.220949 17100 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f: No bootstrap required, opened a new log
05:33:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.253047 17100 raft_consensus.cc:348] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.253894 17100 raft_consensus.cc:374] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:33:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.254240 17100 raft_consensus.cc:729] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 93b85063fc1c405eb2ea9609bb91b01f, State: Initialized, Role: FOLLOWER
05:33:21.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.255205 17100 consensus_queue.cc:260] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:21.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.258378 17204 sys_catalog.cc:455] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:21.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.259335 17204 sys_catalog.cc:458] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [sys.catalog]: This master's current role is: FOLLOWER
05:33:21.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.260298 17100 sys_catalog.cc:564] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [sys.catalog]: configured and running, proceeding with master startup.
05:33:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.260555 17075 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f47357fc8fc9450aae0a29add9dd7731" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "93b85063fc1c405eb2ea9609bb91b01f" is_pre_election: true
05:33:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.261399 17075 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f47357fc8fc9450aae0a29add9dd7731 in term 0.
05:33:21.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.263021 17130 leader_election.cc:302] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 93b85063fc1c405eb2ea9609bb91b01f, f47357fc8fc9450aae0a29add9dd7731; no voters:
05:33:21.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.264330 17189 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 0 FOLLOWER]: Leader pre-election won for term 1
05:33:21.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.264688 17189 raft_consensus.cc:482] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
05:33:21.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.264985 17189 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 0 FOLLOWER]: Advancing to term 1
05:33:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.271613 17189 raft_consensus.cc:504] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.275255 17075 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f47357fc8fc9450aae0a29add9dd7731" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "93b85063fc1c405eb2ea9609bb91b01f"
05:33:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.275830 17075 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [term 0 FOLLOWER]: Advancing to term 1
05:33:21.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.286217 17075 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f47357fc8fc9450aae0a29add9dd7731 in term 1.
05:33:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.287529 17130 leader_election.cc:302] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 93b85063fc1c405eb2ea9609bb91b01f, f47357fc8fc9450aae0a29add9dd7731; no voters:
05:33:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.292265 17189 leader_election.cc:288] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [CANDIDATE]: Term 1 election: Requested vote from peers f01dd8bbee7e4b808226bda4398c0581 (127.16.140.126:42451), 93b85063fc1c405eb2ea9609bb91b01f (127.16.140.125:35207)
05:33:21.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.295009 17189 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 1 FOLLOWER]: Leader election won for term 1
05:33:21.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.297191 17189 raft_consensus.cc:686] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [term 1 LEADER]: Becoming Leader. State: Replica: f47357fc8fc9450aae0a29add9dd7731, State: Running, Role: LEADER
05:33:21.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.300858 17189 consensus_queue.cc:237] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:21.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.321029 17212 sys_catalog.cc:455] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f47357fc8fc9450aae0a29add9dd7731. Latest consensus state: current_term: 1 leader_uuid: "f47357fc8fc9450aae0a29add9dd7731" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:21.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.322404 17212 sys_catalog.cc:458] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: This master's current role is: LEADER
05:33:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.325023 17218 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
05:33:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.326460 17217 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
05:33:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.333002 17218 catalog_manager.cc:1484] Initializing Kudu cluster ID...
05:33:21.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.353621 17075 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [term 1 FOLLOWER]: Refusing update from remote peer f47357fc8fc9450aae0a29add9dd7731: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:33:21.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.360503 17189 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [LEADER]: Connected to new peer: Peer: permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:33:21.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.330416 17027 master_runner.cc:186] Error getting master registration for 127.16.140.126:42451: OK, instance_id { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" instance_seqno: 1790314396964027 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:33:21.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.363548 17027 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:33:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.386065 17204 sys_catalog.cc:455] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f47357fc8fc9450aae0a29add9dd7731. Latest consensus state: current_term: 1 leader_uuid: "f47357fc8fc9450aae0a29add9dd7731" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:21.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.387176 17204 sys_catalog.cc:458] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [sys.catalog]: This master's current role is: FOLLOWER
05:33:21.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.388211 17219 mvcc.cc:205] Tried to move back new op lower bound from 7333127787921256448 to 7333127787767742464. Current Snapshot: MvccSnapshot[applied={T|T < 7333127787921256448}]
05:33:21.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.390486 17189 sys_catalog.cc:455] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f47357fc8fc9450aae0a29add9dd7731" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:21.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.391618 17189 sys_catalog.cc:458] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: This master's current role is: LEADER
05:33:21.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.394042 17218 catalog_manager.cc:1347] Generated new cluster ID: ca9a9f627e9046529d548e5d94b0edd8
05:33:21.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.394428 17218 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
05:33:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.396067 17204 sys_catalog.cc:455] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f47357fc8fc9450aae0a29add9dd7731" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.396871 17204 sys_catalog.cc:458] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f [sys.catalog]: This master's current role is: FOLLOWER
05:33:21.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.432233 17218 catalog_manager.cc:1370] Generated new certificate authority record
05:33:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.434859 17218 catalog_manager.cc:1504] Loading token signing keys...
05:33:21.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.457500 17218 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731: Generated new TSK 0
05:33:21.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.459250 17218 catalog_manager.cc:1514] Initializing in-progress tserver states...
05:33:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.486182 16945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/wal
05:33:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/data
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/logs
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/data/info.pb
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.65:0
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.140.65
05:33:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.65
05:33:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:33:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.632066 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } attempt: 7
05:33:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.677062 17025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581: Bootstrap starting.
05:33:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.693377 17025 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581: Neither blocks nor log segments found. Creating new log.
05:33:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.696018 17025 log.cc:824] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581: Log is configured to *not* fsync() on all Append() calls
05:33:21.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.709367 17025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581: No bootstrap required, opened a new log
05:33:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.769073 17025 raft_consensus.cc:348] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.770391 17025 raft_consensus.cc:374] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:33:21.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.770808 17025 raft_consensus.cc:729] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f01dd8bbee7e4b808226bda4398c0581, State: Initialized, Role: FOLLOWER
05:33:21.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.771836 17025 consensus_queue.cc:260] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } }
05:33:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.775262 17227 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.780314 17227 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [sys.catalog]: This master's current role is: FOLLOWER
05:33:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.777928 17025 sys_catalog.cc:564] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [sys.catalog]: configured and running, proceeding with master startup.
05:33:21.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.779474 16999 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f47357fc8fc9450aae0a29add9dd7731" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f01dd8bbee7e4b808226bda4398c0581"
05:33:21.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.780128 17000 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f47357fc8fc9450aae0a29add9dd7731" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f01dd8bbee7e4b808226bda4398c0581" is_pre_election: true
05:33:21.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.778987 16998 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [term 0 FOLLOWER]: Advancing to term 1
05:33:21.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.790442 16998 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [term 1 FOLLOWER]: Refusing update from remote peer f47357fc8fc9450aae0a29add9dd7731: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:33:21.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.792850 17212 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:33:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.849660 17227 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f47357fc8fc9450aae0a29add9dd7731. Latest consensus state: current_term: 1 leader_uuid: "f47357fc8fc9450aae0a29add9dd7731" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.850409 17227 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [sys.catalog]: This master's current role is: FOLLOWER
05:33:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.849821 17236 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f47357fc8fc9450aae0a29add9dd7731" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.851240 17236 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581 [sys.catalog]: This master's current role is: FOLLOWER
05:33:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.886701 17243 catalog_manager.cc:1259] Loaded cluster ID: ca9a9f627e9046529d548e5d94b0edd8
05:33:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.886958 17243 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581: loading cluster ID for follower catalog manager: success
05:33:21.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.905097 17243 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581: acquiring CA information for follower catalog manager: success
05:33:21.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:21.915354 17243 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f01dd8bbee7e4b808226bda4398c0581: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:33:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:21.972506 17224 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:21.973146 17224 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:21.973887 17224 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:22.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:22.033769 17224 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:22.036612 17224 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.140.65
05:33:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.100376 17224 tablet_server_runner.cc:78] Tablet server non-default flags:
05:33:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/data
05:33:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/wal
05:33:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.65:0
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/data/info.pb
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.65
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.140.65
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/logs
05:33:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.101936 17224 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:22.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.104017 17224 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:22.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:22.127856 17250 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.355392 17217 catalog_manager.cc:1259] Loaded cluster ID: ca9a9f627e9046529d548e5d94b0edd8
05:33:22.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.364399 17217 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f: loading cluster ID for follower catalog manager: success
05:33:22.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.388453 17217 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f: acquiring CA information for follower catalog manager: success
05:33:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.394662 17217 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 93b85063fc1c405eb2ea9609bb91b01f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:33:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.426393 17212 sys_catalog.cc:455] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f47357fc8fc9450aae0a29add9dd7731" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f01dd8bbee7e4b808226bda4398c0581" member_type: VOTER last_known_addr { host: "127.16.140.126" port: 42451 } } peers { permanent_uuid: "93b85063fc1c405eb2ea9609bb91b01f" member_type: VOTER last_known_addr { host: "127.16.140.125" port: 35207 } } peers { permanent_uuid: "f47357fc8fc9450aae0a29add9dd7731" member_type: VOTER last_known_addr { host: "127.16.140.124" port: 40427 } } }
05:33:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:22.427170 17212 sys_catalog.cc:458] T 00000000000000000000000000000000 P f47357fc8fc9450aae0a29add9dd7731 [sys.catalog]: This master's current role is: LEADER
05:33:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:23.524508 17249 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 17224
05:33:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:24.375764 17224 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.250s user 0.795s sys 1.253s
05:33:24.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:24.376088 17224 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.250s user 0.795s sys 1.253s
05:33:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:22.128800 17251 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:24.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:24.391867 17257 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:24.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.392167 17224 server_base.cc:1034] running on GCE node
05:33:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.394140 17224 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:24.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.396806 17224 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:24.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.398239 17224 hybrid_clock.cc:648] HybridClock initialized: now 1790314404398184 us; error 60 us; skew 500 ppm
05:33:24.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.399289 17224 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.410830 17224 webserver.cc:460] Webserver started at http://127.16.140.65:39985/ using document root <none> and password file <none>
05:33:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.412125 17224 fs_manager.cc:359] Metadata directory not provided
05:33:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.412398 17224 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.412942 17224 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.420038 17224 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/data/instance:
05:33:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d112715896d440c98bf209d3de994a0"
05:33:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:24 on dist-test-slave-w8hg"
05:33:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.421651 17224 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/wal/instance:
05:33:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d112715896d440c98bf209d3de994a0"
05:33:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:24 on dist-test-slave-w8hg"
05:33:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.431260 17224 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.009s sys 0.001s
05:33:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.439013 17263 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.440411 17224 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
05:33:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.440788 17224 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/wal
05:33:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d112715896d440c98bf209d3de994a0"
05:33:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:24 on dist-test-slave-w8hg"
05:33:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.441244 17224 fs_report.cc:383] FS layout report
05:33:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/wal
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/wal
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/data/data
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:24.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.542893 17224 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:24.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.544899 17224 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:24.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.545481 17224 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.548923 17224 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:33:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.554790 17224 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:33:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.555079 17224 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:33:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.555365 17224 ts_tablet_manager.cc:610] Registered 0 tablets
05:33:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.555572 17224 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:33:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.774689 17224 rpc_server.cc:307] RPC server started. Bound to: 127.16.140.65:35215
05:33:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.778354 17224 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/data/info.pb
05:33:24.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.780722 16945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17224
05:33:24.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.781183 16945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-0/wal/instance
05:33:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.804859 16945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/wal
05:33:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/data
05:33:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:24.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:24.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:24.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:24.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:24.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/logs
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/data/info.pb
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.66:0
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.140.66
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.66
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:33:24.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.774820 17375 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.140.65:35215 every 8 connection(s)
05:33:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.851550 17376 heartbeater.cc:347] Connected to a master server at 127.16.140.124:40427
05:33:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.852030 17376 heartbeater.cc:464] Registering TS with master...
05:33:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.853300 17376 heartbeater.cc:511] Master 127.16.140.124:40427 requested a full tablet report, sending...
05:33:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.856360 17143 ts_manager.cc:194] Registered new tserver with Master: 6d112715896d440c98bf209d3de994a0 (127.16.140.65:35215)
05:33:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.859272 17143 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.140.65:53913
05:33:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.867110 17379 heartbeater.cc:347] Connected to a master server at 127.16.140.125:35207
05:33:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.867450 17379 heartbeater.cc:464] Registering TS with master...
05:33:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.868178 17379 heartbeater.cc:511] Master 127.16.140.125:35207 requested a full tablet report, sending...
05:33:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.871459 17065 ts_manager.cc:194] Registered new tserver with Master: 6d112715896d440c98bf209d3de994a0 (127.16.140.65:35215)
05:33:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.877758 17380 heartbeater.cc:347] Connected to a master server at 127.16.140.126:42451
05:33:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.878103 17380 heartbeater.cc:464] Registering TS with master...
05:33:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.878757 17380 heartbeater.cc:511] Master 127.16.140.126:42451 requested a full tablet report, sending...
05:33:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:24.881618 16990 ts_manager.cc:194] Registered new tserver with Master: 6d112715896d440c98bf209d3de994a0 (127.16.140.65:35215)
05:33:25.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:25.278864 17384 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:25.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:25.279433 17384 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:25.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:25.280037 17384 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:25.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:25.313113 17384 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:25.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:25.314069 17384 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.140.66
05:33:25.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:25.350167 17384 tablet_server_runner.cc:78] Tablet server non-default flags:
05:33:25.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:25.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:25.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/data
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/wal
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.66:0
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/data/info.pb
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.66
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.140.66
05:33:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/logs
05:33:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:25.351542 17384 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:25.353178 17384 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:25.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:25.376793 17393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:25.863802 17376 heartbeater.cc:503] Master 127.16.140.124:40427 was elected leader, sending a full tablet report...
05:33:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:25.377210 17394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:25.378334 17396 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:26.584697 17395 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:33:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.584753 17384 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:33:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.589301 17384 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.592042 17384 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.593511 17384 hybrid_clock.cc:648] HybridClock initialized: now 1790314406593469 us; error 60 us; skew 500 ppm
05:33:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.594257 17384 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.600378 17384 webserver.cc:460] Webserver started at http://127.16.140.66:45879/ using document root <none> and password file <none>
05:33:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.601255 17384 fs_manager.cc:359] Metadata directory not provided
05:33:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.601435 17384 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.601830 17384 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.607758 17384 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/data/instance:
05:33:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e00fc14767cb4989860e71a485df5053"
05:33:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:26 on dist-test-slave-w8hg"
05:33:26.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.608951 17384 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/wal/instance:
05:33:26.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e00fc14767cb4989860e71a485df5053"
05:33:26.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:26 on dist-test-slave-w8hg"
05:33:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.615957 17384 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
05:33:26.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.621737 17405 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.622851 17384 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.623152 17384 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/wal
05:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e00fc14767cb4989860e71a485df5053"
05:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:26 on dist-test-slave-w8hg"
05:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.623473 17384 fs_report.cc:383] FS layout report
05:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/wal
05:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/wal
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/data/data
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:26.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.667728 17384 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:26.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.669268 17384 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:26.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.669750 17384 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:26.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.672755 17384 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:33:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.676931 17384 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:33:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.677183 17384 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:33:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.677418 17384 ts_tablet_manager.cc:610] Registered 0 tablets
05:33:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.677548 17384 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:33:26.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.830209 17384 rpc_server.cc:307] RPC server started. Bound to: 127.16.140.66:42529
05:33:26.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.830440 17517 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.140.66:42529 every 8 connection(s)
05:33:26.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.833367 17384 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/data/info.pb
05:33:26.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.842634 16945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17384
05:33:26.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.843426 16945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-1/wal/instance
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.866961 16945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/wal
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/data
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/logs
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/data/info.pb
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.67:0
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.140.67
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.67
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:33:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.885987 17520 heartbeater.cc:347] Connected to a master server at 127.16.140.126:42451
05:33:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.886267 17518 heartbeater.cc:347] Connected to a master server at 127.16.140.124:40427
05:33:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.886538 17520 heartbeater.cc:464] Registering TS with master...
05:33:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.886667 17518 heartbeater.cc:464] Registering TS with master...
05:33:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.887903 17519 heartbeater.cc:347] Connected to a master server at 127.16.140.125:35207
05:33:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.887977 17520 heartbeater.cc:511] Master 127.16.140.126:42451 requested a full tablet report, sending...
05:33:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.888267 17519 heartbeater.cc:464] Registering TS with master...
05:33:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.889048 17519 heartbeater.cc:511] Master 127.16.140.125:35207 requested a full tablet report, sending...
05:33:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.890399 17518 heartbeater.cc:511] Master 127.16.140.124:40427 requested a full tablet report, sending...
05:33:26.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.891289 16990 ts_manager.cc:194] Registered new tserver with Master: e00fc14767cb4989860e71a485df5053 (127.16.140.66:42529)
05:33:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.893427 17065 ts_manager.cc:194] Registered new tserver with Master: e00fc14767cb4989860e71a485df5053 (127.16.140.66:42529)
05:33:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.893676 17143 ts_manager.cc:194] Registered new tserver with Master: e00fc14767cb4989860e71a485df5053 (127.16.140.66:42529)
05:33:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:26.896076 17143 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.140.66:40643
05:33:27.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:27.181990 17528 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:27.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:27.182556 17528 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:27.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:27.183142 17528 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:27.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:27.214023 17528 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:27.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:27.214990 17528 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.140.67
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:27.248723 17528 tablet_server_runner.cc:78] Tablet server non-default flags:
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.140.84:42611
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/data
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/wal
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.140.67:0
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/data/info.pb
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.140.67
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.140.126:42451,127.16.140.125:35207,127.16.140.124:40427
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.140.67
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/logs
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:27.250288 17528 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:27.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:27.251962 17528 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:27.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:27.271961 17535 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:27.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:27.900269 17518 heartbeater.cc:503] Master 127.16.140.124:40427 was elected leader, sending a full tablet report...
05:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:27.272914 17537 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:27.272006 17534 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:28.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:28.463928 17536 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:33:28.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.464064 17528 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:33:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.468401 17528 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.470595 17528 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:28.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.471925 17528 hybrid_clock.cc:648] HybridClock initialized: now 1790314408471807 us; error 139 us; skew 500 ppm
05:33:28.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.472748 17528 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:28.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.478989 17528 webserver.cc:460] Webserver started at http://127.16.140.67:34227/ using document root <none> and password file <none>
05:33:28.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.479861 17528 fs_manager.cc:359] Metadata directory not provided
05:33:28.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.480154 17528 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:28.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.480660 17528 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.486038 17528 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/data/instance:
05:33:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd11cf2e7d624cd5934a03a2d2c33a5c"
05:33:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:28 on dist-test-slave-w8hg"
05:33:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.487200 17528 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/wal/instance:
05:33:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd11cf2e7d624cd5934a03a2d2c33a5c"
05:33:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:28 on dist-test-slave-w8hg"
05:33:28.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.494279 17528 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.003s
05:33:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.499600 17547 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:28.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.501233 17528 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
05:33:28.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.501582 17528 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/wal
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd11cf2e7d624cd5934a03a2d2c33a5c"
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:28 on dist-test-slave-w8hg"
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.501953 17528 fs_report.cc:383] FS layout report
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/wal
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/wal
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/data/data
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:28.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:28.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:28.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.555506 17528 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:28.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.557152 17528 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:28.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.557717 17528 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.560292 17528 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:33:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.564487 17528 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:33:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.564787 17528 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:33:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.565102 17528 ts_tablet_manager.cc:610] Registered 0 tablets
05:33:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.565301 17528 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:33:28.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.724613 17528 rpc_server.cc:307] RPC server started. Bound to: 127.16.140.67:42227
05:33:28.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.724752 17659 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.140.67:42227 every 8 connection(s)
05:33:28.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.727164 17528 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/data/info.pb
05:33:28.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.727838 16945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17528
05:33:28.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.728260 16945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster6907512176926872132/ts-2/wal/instance
05:33:28.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.784174 17662 heartbeater.cc:347] Connected to a master server at 127.16.140.126:42451
05:33:28.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.784457 17661 heartbeater.cc:347] Connected to a master server at 127.16.140.125:35207
05:33:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.784693 17662 heartbeater.cc:464] Registering TS with master...
05:33:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.784804 17661 heartbeater.cc:464] Registering TS with master...
05:33:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.786067 17661 heartbeater.cc:511] Master 127.16.140.125:35207 requested a full tablet report, sending...
05:33:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.786027 17662 heartbeater.cc:511] Master 127.16.140.126:42451 requested a full tablet report, sending...
05:33:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.789453 17065 ts_manager.cc:194] Registered new tserver with Master: fd11cf2e7d624cd5934a03a2d2c33a5c (127.16.140.67:42227)
05:33:28.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.789893 16990 ts_manager.cc:194] Registered new tserver with Master: fd11cf2e7d624cd5934a03a2d2c33a5c (127.16.140.67:42227)
05:33:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.793010 17660 heartbeater.cc:347] Connected to a master server at 127.16.140.124:40427
05:33:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.793339 17660 heartbeater.cc:464] Registering TS with master...
05:33:28.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.794123 17660 heartbeater.cc:511] Master 127.16.140.124:40427 requested a full tablet report, sending...
05:33:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.796000 17143 ts_manager.cc:194] Registered new tserver with Master: fd11cf2e7d624cd5934a03a2d2c33a5c (127.16.140.67:42227)
05:33:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.798364 17143 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.140.67:46251
05:33:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:28.814534 16945 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
05:33:28.828 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:33:28.830 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
05:33:28.843 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.140.126"
port: 42451
}
bound_http_address {
host: "127.16.140.126"
port: 36387
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.140.125"
port: 35207
}
bound_http_address {
host: "127.16.140.125"
port: 44053
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.140.124"
port: 40427
}
bound_http_address {
host: "127.16.140.124"
port: 40685
}
}
}
05:33:28.849 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:33:28.852 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.16.140.65"
port: 35215
}
bound_http_address {
host: "127.16.140.65"
port: 39985
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.16.140.66"
port: 42529
}
bound_http_address {
host: "127.16.140.66"
port: 45879
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.16.140.67"
port: 42227
}
bound_http_address {
host: "127.16.140.67"
port: 34227
}
}
}
05:33:28.852 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
05:33:29.047 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
05:33:29.177 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskSNHo_2/test-tmp
05:33:29.178 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:33:29.178 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskSNHo_2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:33:29.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
05:33:29.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:29.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:33:29.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:29.530831 17673 mini_oidc.cc:152] Starting JWKS server
05:33:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:29.534339 17673 webserver.cc:460] Webserver started at http://127.0.0.1:44991/ using document root <none> and password file <none>
05:33:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:29.534721 17673 mini_oidc.cc:189] Starting OIDC Discovery server
05:33:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:29.536342 17673 webserver.cc:460] Webserver started at http://127.0.0.1:35421/ using document root <none> and password file <none>
05:33:29.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
05:33:29.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/principal' for realm 'KRBTEST.COM',
05:33:29.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
05:33:29.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:29 dist-test-slave-w8hg krb5kdc[17685](info): setting up network...
05:33:29.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
05:33:29.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:29 dist-test-slave-w8hg krb5kdc[17685](info): set up 2 sockets
05:33:29.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:29 dist-test-slave-w8hg krb5kdc[17685](info): commencing operation
05:33:29.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
05:33:29.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:29.801939 17660 heartbeater.cc:503] Master 127.16.140.124:40427 was elected leader, sending a full tablet report...
05:33:30.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:30.660271 17673 mini_kdc.cc:121] Time spent starting KDC: real 1.100s user 0.004s sys 0.020s
05:33:30.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
05:33:30.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:33:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
05:33:30.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
05:33:30.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:33:30.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
05:33:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
05:33:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:33:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
05:33:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:33:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/test-user.keytab.
05:33:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/test-user.keytab.
05:33:30.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:30 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314410, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:30.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:33:30.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:33:30Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:33:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:33:30Z Disabled control of system clock
05:33:30.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.66.126@KRBTEST.COM; defaulting to no policy
05:33:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:33:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.66.126@KRBTEST.COM" created.
05:33:30.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:33:30.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.66.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.126.keytab.
05:33:30.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.66.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.126.keytab.
05:33:30.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:30.873314 17673 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:30.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:30.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:30.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data
05:33:30.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/logs
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/info.pb
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.126:42991
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.126
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.126.keytab
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.126
05:33:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:33:31.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.214257 17698 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.216156 17698 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.217278 17698 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:31.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.254424 17698 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:33:31.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.254655 17698 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.254906 17698 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.255174 17698 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.255373 17698 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.255561 17698 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.255699 17698 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44991/jwks.json
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:31.292981 17698 master_runner.cc:386] Master server non-default flags:
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.126.keytab
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.126
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.126:42991
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/info.pb
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.126
05:33:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/logs
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:31.294081 17698 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:31.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:31.295097 17698 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.314741 17705 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.315475 17707 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:31.318079 17698 server_base.cc:1034] running on GCE node
05:33:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:31.315469 17704 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:32.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.536291 17698 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.539399 17698 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.540828 17698 hybrid_clock.cc:648] HybridClock initialized: now 1790314412540791 us; error 112 us; skew 500 ppm
05:33:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.541592 17698 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:32.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:32 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314412, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:32.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.547621 17698 init.cc:377] Logged in from keytab as kudu/127.17.66.126@KRBTEST.COM (short username kudu)
05:33:32.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.553294 17698 webserver.cc:460] Webserver started at http://127.17.66.126:36881/ using document root <none> and password file <none>
05:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.554147 17698 fs_manager.cc:359] Metadata directory not provided
05:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.554318 17698 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.554699 17698 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:32.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.559449 17698 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/instance:
05:33:32.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "397672e383b244f5899681f25c7acf35"
05:33:32.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:32 on dist-test-slave-w8hg"
05:33:32.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.560578 17698 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal/instance:
05:33:32.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "397672e383b244f5899681f25c7acf35"
05:33:32.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:32 on dist-test-slave-w8hg"
05:33:32.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.568560 17698 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:33:32.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.574013 17716 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:32.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.575117 17698 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
05:33:32.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.575389 17698 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:32.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "397672e383b244f5899681f25c7acf35"
05:33:32.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:32 on dist-test-slave-w8hg"
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.575769 17698 fs_report.cc:383] FS layout report
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/data
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:32.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.612736 17698 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.618484 17698 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:32.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.618891 17698 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.701174 17698 rpc_server.cc:307] RPC server started. Bound to: 127.17.66.126:42991
05:33:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.701237 17767 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.66.126:42991 every 8 connection(s)
05:33:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.703769 17698 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/info.pb
05:33:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.708840 17673 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17698
05:33:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.709383 17673 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal/instance
05:33:32.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.711076 17768 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:33:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.66.125@KRBTEST.COM; defaulting to no policy
05:33:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.729532 17768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } has no permanent_uuid. Determining permanent_uuid...
05:33:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:33:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.66.125@KRBTEST.COM" created.
05:33:32.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:32 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314412, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.126@KRBTEST.COM for kudu/127.17.66.126@KRBTEST.COM
05:33:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:33:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.66.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.125.keytab.
05:33:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.66.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.125.keytab.
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.762988 17673 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/logs
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/info.pb
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.125:35429
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.125
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.125.keytab
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.125
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:33:32.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.774519 17768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } has no permanent_uuid. Determining permanent_uuid...
05:33:32.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:32.777405 17717 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:32.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:32.782639 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.125:35429: Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:32.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.832162 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } attempt: 1
05:33:32.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:32.836436 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.125:35429: Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.904816 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } attempt: 2
05:33:32.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:32.908185 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.125:35429: Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:33.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:32.999631 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } attempt: 3
05:33:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.004225 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.125:35429: Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.097047 17774 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.097738 17774 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:33.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.098440 17774 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.130435 17774 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:33:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.130716 17774 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.131091 17774 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.131410 17774 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.131713 17774 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:33.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.132022 17774 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:33.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.132287 17774 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44991/jwks.json
05:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:33.147714 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } attempt: 4
05:33:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.151098 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.125:35429: Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:33.169991 17774 master_runner.cc:386] Master server non-default flags:
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.125.keytab
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.125
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.125:35429
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/info.pb
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.125
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/logs
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:33.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:33.171509 17774 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:33.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:33.172835 17774 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.192485 17781 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:33.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:33.450479 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } attempt: 5
05:33:33.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.455304 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.125:35429: Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:34.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.002902 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } attempt: 6
05:33:34.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:34.010931 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.125:35429: Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:34.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.192648 17782 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:34.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:33.195001 17774 server_base.cc:1034] running on GCE node
05:33:34.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:33.193884 17784 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.422622 17774 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.425567 17774 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:34.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.426950 17774 hybrid_clock.cc:648] HybridClock initialized: now 1790314414426919 us; error 41 us; skew 500 ppm
05:33:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.427811 17774 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:34.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:34 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314414, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:34.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.433519 17774 init.cc:377] Logged in from keytab as kudu/127.17.66.125@KRBTEST.COM (short username kudu)
05:33:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.439002 17774 webserver.cc:460] Webserver started at http://127.17.66.125:34067/ using document root <none> and password file <none>
05:33:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.439899 17774 fs_manager.cc:359] Metadata directory not provided
05:33:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.440141 17774 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.440655 17774 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.445411 17774 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/instance:
05:33:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69a963369cde45378346599392aa251d"
05:33:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:34 on dist-test-slave-w8hg"
05:33:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.446538 17774 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal/instance:
05:33:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69a963369cde45378346599392aa251d"
05:33:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:34 on dist-test-slave-w8hg"
05:33:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.454129 17774 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
05:33:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.459920 17794 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.461120 17774 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:33:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.461517 17774 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69a963369cde45378346599392aa251d"
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:34 on dist-test-slave-w8hg"
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.461869 17774 fs_report.cc:383] FS layout report
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/data
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:34.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.500577 17774 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:34.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.506093 17774 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:34.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.506626 17774 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:34.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.589568 17774 rpc_server.cc:307] RPC server started. Bound to: 127.17.66.125:35429
05:33:34.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.589716 17845 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.66.125:35429 every 8 connection(s)
05:33:34.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.592206 17774 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/info.pb
05:33:34.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.597424 17673 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17774
05:33:34.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.597890 17673 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal/instance
05:33:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.599081 17846 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:33:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.614105 17846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } has no permanent_uuid. Determining permanent_uuid...
05:33:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.66.124@KRBTEST.COM; defaulting to no policy
05:33:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:33:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.66.124@KRBTEST.COM" created.
05:33:34.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:33:34.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.66.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.124.keytab.
05:33:34.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.66.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.124.keytab.
05:33:34.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.644618 17673 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/logs
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/info.pb
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.124:39483
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.124
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.124.keytab
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.124
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:33:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:34 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314414, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.125@KRBTEST.COM for kudu/127.17.66.126@KRBTEST.COM
05:33:34.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.661821 17846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } has no permanent_uuid. Determining permanent_uuid...
05:33:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:34 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314414, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.125@KRBTEST.COM for kudu/127.17.66.125@KRBTEST.COM
05:33:34.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.691258 17846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } has no permanent_uuid. Determining permanent_uuid...
05:33:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:34.693426 17797 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:34.697850 17846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.747370 17846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 1
05:33:34.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:34.751097 17846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:34.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.819525 17846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 2
05:33:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:34.832497 17846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:34.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:34.923923 17846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 3
05:33:34.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:34.927879 17846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.003964 17852 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.004741 17852 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.005739 17852 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.042419 17852 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:33:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.042644 17852 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.042953 17852 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.043260 17852 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.043546 17852 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.043751 17852 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.043892 17852 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44991/jwks.json
05:33:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.071319 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } attempt: 7
05:33:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.071314 17846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 4
05:33:35.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.076151 17846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:35 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314412, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.126@KRBTEST.COM for kudu/127.17.66.125@KRBTEST.COM
05:33:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.087344 17852 master_runner.cc:386] Master server non-default flags:
05:33:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.124.keytab
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.124
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.124:39483
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/info.pb
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.124
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/logs
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.088651 17852 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.089707 17852 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:35.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.099202 17768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } has no permanent_uuid. Determining permanent_uuid...
05:33:35.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.106817 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.113309 17861 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:35.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.165226 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 1
05:33:35.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.168859 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.250427 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 2
05:33:35.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.255611 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.341190 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 3
05:33:35.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.345906 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.375523 17846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 5
05:33:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.379596 17846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.486306 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 4
05:33:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.490610 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.774075 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 5
05:33:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.779067 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:35.927017 17846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 6
05:33:35.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.935262 17846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.331468 17768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } attempt: 6
05:33:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:36.338301 17768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.66.124:39483: Network error: Client connection negotiation failed: client connection to 127.17.66.124:39483: connect: Connection refused (error 111)
05:33:36.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.114703 17862 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:36.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:35.115180 17865 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:36.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:36.373715 17864 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:33:36.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.373842 17852 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:33:36.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.377753 17852 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.380813 17852 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.382227 17852 hybrid_clock.cc:648] HybridClock initialized: now 1790314416382195 us; error 51 us; skew 500 ppm
05:33:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.382997 17852 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314416, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.389019 17852 init.cc:377] Logged in from keytab as kudu/127.17.66.124@KRBTEST.COM (short username kudu)
05:33:36.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.395628 17852 webserver.cc:460] Webserver started at http://127.17.66.124:39163/ using document root <none> and password file <none>
05:33:36.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.396579 17852 fs_manager.cc:359] Metadata directory not provided
05:33:36.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.396759 17852 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:36.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.397123 17852 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:33:36.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.402143 17852 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/instance:
05:33:36.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25c80047dd1d4619acdbbf4cffffcf6a"
05:33:36.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:36 on dist-test-slave-w8hg"
05:33:36.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.403540 17852 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal/instance:
05:33:36.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25c80047dd1d4619acdbbf4cffffcf6a"
05:33:36.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:36 on dist-test-slave-w8hg"
05:33:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.410985 17852 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
05:33:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.416800 17880 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.418107 17852 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
05:33:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.418448 17852 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25c80047dd1d4619acdbbf4cffffcf6a"
05:33:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:36 on dist-test-slave-w8hg"
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.418736 17852 fs_report.cc:383] FS layout report
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/data
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.469920 17852 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.475643 17852 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.476094 17852 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:36.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.560161 17852 rpc_server.cc:307] RPC server started. Bound to: 127.17.66.124:39483
05:33:36.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.560262 17931 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.66.124:39483 every 8 connection(s)
05:33:36.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.563138 17852 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/info.pb
05:33:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.568662 17932 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:33:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.573618 17673 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17852
05:33:36.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.574060 17673 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal/instance
05:33:36.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.582140 17932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } has no permanent_uuid. Determining permanent_uuid...
05:33:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314416, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.124@KRBTEST.COM for kudu/127.17.66.126@KRBTEST.COM
05:33:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314410, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.66.124@KRBTEST.COM
05:33:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.625965 17932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } has no permanent_uuid. Determining permanent_uuid...
05:33:36.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314416, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.124@KRBTEST.COM for kudu/127.17.66.125@KRBTEST.COM
05:33:36.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314410, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.66.125@KRBTEST.COM
05:33:36.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.656915 17932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } has no permanent_uuid. Determining permanent_uuid...
05:33:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314416, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.124@KRBTEST.COM for kudu/127.17.66.124@KRBTEST.COM
05:33:36.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314410, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.66.126@KRBTEST.COM
05:33:36.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.688293 17932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: Bootstrap starting.
05:33:36.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.694761 17673 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
05:33:36.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.694794 17932 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: Neither blocks nor log segments found. Creating new log.
05:33:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.696646 17932 log.cc:824] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: Log is configured to *not* fsync() on all Append() calls
05:33:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.701606 17932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: No bootstrap required, opened a new log
05:33:36.703 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:36.705 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
05:33:36.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
05:33:36.720 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
05:33:36.726 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.66.126"
port: 42991
}
bound_http_address {
host: "127.17.66.126"
port: 36881
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.66.125"
port: 35429
}
bound_http_address {
host: "127.17.66.125"
port: 34067
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.66.124"
port: 39483
}
bound_http_address {
host: "127.17.66.124"
port: 39163
}
}
}
05:33:36.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:33:36.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:36.727 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:33:36.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:33:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.729647 17932 raft_consensus.cc:348] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.730410 17932 raft_consensus.cc:374] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:33:36.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.730693 17932 raft_consensus.cc:729] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 25c80047dd1d4619acdbbf4cffffcf6a, State: Initialized, Role: FOLLOWER
05:33:36.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.731576 17932 consensus_queue.cc:260] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.739079 17941 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } } }
05:33:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.739923 17941 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [sys.catalog]: This master's current role is: FOLLOWER
05:33:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.740849 17932 sys_catalog.cc:564] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [sys.catalog]: configured and running, proceeding with master startup.
05:33:36.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314416, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:36.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:33:36.773 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:36.775 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc
05:33:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:36.780258 17954 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:36.780707 17954 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:36.785 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:33:36.786 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc
05:33:36.789 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:33:36.792 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
05:33:36.794 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:33:36.794 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:33:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.767309 17852 master_runner.cc:186] Error getting master registration for 127.17.66.126:42991: OK, instance_id { permanent_uuid: "397672e383b244f5899681f25c7acf35" instance_seqno: 1790314412678612 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:33:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.804440 17852 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:33:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:36 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314416, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:36.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:33:36.835 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:36.835 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc
05:33:36.837 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:33:36.838 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.66.125:35429
05:33:36.844 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
05:33:36.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.845202 17673 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 17774
05:33:36.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:36.866 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.66.124:39483
05:33:36.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
05:33:36.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.867241 17673 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 17852
05:33:36.888 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:36.888 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.66.126:42991
05:33:36.888 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
05:33:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.889251 17673 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 17698
05:33:36.912 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:36.912 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.66.125:35429
05:33:36.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
05:33:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:36.915417 17673 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/logs
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/info.pb
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.125:35429
05:33:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.125
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34067
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.125.keytab
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.125
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:33:37.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.228223 17958 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.228929 17958 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.229544 17958 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.259690 17958 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:33:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.259907 17958 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.260179 17958 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.260372 17958 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.260679 17958 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.260883 17958 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.261019 17958 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44991/jwks.json
05:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:37.297981 17958 master_runner.cc:386] Master server non-default flags:
05:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data
05:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.125.keytab
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.125
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.125:35429
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/info.pb
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.125
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34067
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/logs
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:37.299127 17958 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:37.300053 17958 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:37.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.319190 17965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.319289 17966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:38.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:37.319407 17968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:38.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:37.324465 17958 server_base.cc:1034] running on GCE node
05:33:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.537988 17958 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:38.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.540725 17958 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.542141 17958 hybrid_clock.cc:648] HybridClock initialized: now 1790314418542102 us; error 52 us; skew 500 ppm
05:33:38.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.543463 17958 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:38.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:38 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314418, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.551215 17958 init.cc:377] Logged in from keytab as kudu/127.17.66.125@KRBTEST.COM (short username kudu)
05:33:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.557653 17958 webserver.cc:460] Webserver started at http://127.17.66.125:34067/ using document root <none> and password file <none>
05:33:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.558956 17958 fs_manager.cc:359] Metadata directory not provided
05:33:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.559211 17958 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.567153 17958 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
05:33:38.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.571705 17977 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:38.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.572851 17958 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:33:38.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.573139 17958 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:38.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69a963369cde45378346599392aa251d"
05:33:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:34 on dist-test-slave-w8hg"
05:33:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.575464 17958 fs_report.cc:383] FS layout report
05:33:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal
05:33:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/data
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:38.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.617656 17958 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:38.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.623720 17958 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:38.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.624209 17958 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:38.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.696611 17958 rpc_server.cc:307] RPC server started. Bound to: 127.17.66.125:35429
05:33:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.696758 18028 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.66.125:35429 every 8 connection(s)
05:33:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.699398 17958 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/data/info.pb
05:33:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.705276 17673 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 17958
05:33:38.705 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:38.706 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.66.124:39483
05:33:38.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
05:33:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.707315 17673 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data
05:33:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/logs
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/info.pb
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.124:39483
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.124
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39163
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.124.keytab
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.124
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:38.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:38.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:38.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:33:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.710857 18029 sys_catalog.cc:263] Verifying existing consensus state
05:33:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260925 05:33:38.711619 18029 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:33:38.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.712067 17958 master.cc:537] Master@127.17.66.125:35429 shutting down...
05:33:38.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:38.776985 17958 master.cc:552] Master@127.17.66.125:35429 shutdown complete.
05:33:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:33:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.045018 18031 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.045610 18031 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.046203 18031 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:39.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.076913 18031 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:33:39.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.077136 18031 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:39.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.077535 18031 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:39.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.077793 18031 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.078092 18031 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.078405 18031 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.078670 18031 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44991/jwks.json
05:33:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:39.114228 18031 master_runner.cc:386] Master server non-default flags:
05:33:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.124.keytab
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.124
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.124:39483
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/info.pb
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.124
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39163
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/logs
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:39.115435 18031 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:39.116765 18031 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.135818 18042 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:40.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.135866 18039 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:40.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:39.137499 18031 server_base.cc:1034] running on GCE node
05:33:40.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:39.137419 18040 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.364463 18031 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.366979 18031 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.368387 18031 hybrid_clock.cc:648] HybridClock initialized: now 1790314420368337 us; error 51 us; skew 500 ppm
05:33:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.369131 18031 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:40.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:40 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314420, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:40.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.379876 18031 init.cc:377] Logged in from keytab as kudu/127.17.66.124@KRBTEST.COM (short username kudu)
05:33:40.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.384529 18031 webserver.cc:460] Webserver started at http://127.17.66.124:39163/ using document root <none> and password file <none>
05:33:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.385440 18031 fs_manager.cc:359] Metadata directory not provided
05:33:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.385632 18031 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:40.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.392988 18031 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
05:33:40.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.397125 18053 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:40.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.398104 18031 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
05:33:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.398375 18031 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25c80047dd1d4619acdbbf4cffffcf6a"
05:33:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:36 on dist-test-slave-w8hg"
05:33:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.400511 18031 fs_report.cc:383] FS layout report
05:33:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/wal
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/data
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:40.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:40.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.436224 18031 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.441571 18031 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.442008 18031 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.511096 18031 rpc_server.cc:307] RPC server started. Bound to: 127.17.66.124:39483
05:33:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.511212 18104 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.66.124:39483 every 8 connection(s)
05:33:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.513695 18031 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-2/data/info.pb
05:33:40.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.519618 17673 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 18031
05:33:40.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:40.521 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.66.126:42991
05:33:40.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.523083 17673 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/logs
05:33:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/info.pb
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.126:42991
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.126
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36881
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.126.keytab
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.126
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:40.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:40.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:33:40.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.525379 18105 sys_catalog.cc:263] Verifying existing consensus state
05:33:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.530618 18105 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: Bootstrap starting.
05:33:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.563006 18105 log.cc:824] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: Log is configured to *not* fsync() on all Append() calls
05:33:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.567778 18105 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
05:33:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.568532 18105 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: Bootstrap complete.
05:33:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.593026 18105 raft_consensus.cc:348] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.595274 18105 raft_consensus.cc:374] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:33:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.595568 18105 raft_consensus.cc:729] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 25c80047dd1d4619acdbbf4cffffcf6a, State: Initialized, Role: FOLLOWER
05:33:40.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.596472 18105 consensus_queue.cc:260] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.599459 18109 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } } }
05:33:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.600346 18109 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [sys.catalog]: This master's current role is: FOLLOWER
05:33:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.601094 18105 sys_catalog.cc:564] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [sys.catalog]: configured and running, proceeding with master startup.
05:33:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.630645 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.630976 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.627692 18031 master_runner.cc:186] Error getting master registration for 127.17.66.126:42991: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111),
05:33:40.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.657050 18031 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:33:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.684278 18109 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:33:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.684806 18109 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:40.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.690250 18057 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111)
05:33:40.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.690505 18109 leader_election.cc:288] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991), 69a963369cde45378346599392aa251d (127.17.66.125:35429)
05:33:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.697099 18059 tablet.cc:2369] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:33:40.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.704404 18054 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 69a963369cde45378346599392aa251d (127.17.66.125:35429): Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:40.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.705092 18057 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991): Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111)
05:33:40.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.705521 18057 leader_election.cc:302] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25c80047dd1d4619acdbbf4cffffcf6a; no voters: 397672e383b244f5899681f25c7acf35, 69a963369cde45378346599392aa251d
05:33:40.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.706328 18109 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:33:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.889530 18107 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:33:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.890059 18107 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:33:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.890611 18107 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:33:40.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.923914 18107 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:33:40.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.924240 18107 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:33:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.924681 18107 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:33:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.925014 18107 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:33:40.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.925328 18107 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:33:40.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.925670 18107 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:33:40.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.925921 18107 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44991/jwks.json
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.968021 18107 master_runner.cc:386] Master server non-default flags:
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.66.84:40539
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.66.126:42991,127.17.66.125:35429,127.17.66.124:39483
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/krb5kdc/kudu_127.17.66.126.keytab
05:33:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.66.126
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.66.126:42991
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44991/jwks.json
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/info.pb
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.66.126
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36881
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/logs
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:33:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:33:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:33:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:33:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:33:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:33:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.969529 18107 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:33:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:40.970791 18107 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:33:40.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.990423 18131 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:41.635288 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:41.635628 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.414614 18137 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:33:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.415499 18137 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.432862 18137 leader_election.cc:288] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991), 69a963369cde45378346599392aa251d (127.17.66.125:35429)
05:33:42.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:42.445006 18054 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 69a963369cde45378346599392aa251d (127.17.66.125:35429): Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:42.448241 18057 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991): Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111)
05:33:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.449827 18057 leader_election.cc:302] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25c80047dd1d4619acdbbf4cffffcf6a; no voters: 397672e383b244f5899681f25c7acf35, 69a963369cde45378346599392aa251d
05:33:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.450791 18137 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:33:42.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:42.640249 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:42.640838 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.998616 18133 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.653913 18107 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:33:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:42.653885 18132 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1664 milliseconds
05:33:42.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:40.991637 18130 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:33:42.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.659502 18107 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:33:42.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.662708 18107 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:33:42.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.664172 18107 hybrid_clock.cc:648] HybridClock initialized: now 1790314422664219 us; error 139 us; skew 500 ppm
05:33:42.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.665174 18107 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:33:42.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:42 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314422, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:42.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.672482 18107 init.cc:377] Logged in from keytab as kudu/127.17.66.126@KRBTEST.COM (short username kudu)
05:33:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.678505 18107 webserver.cc:460] Webserver started at http://127.17.66.126:36881/ using document root <none> and password file <none>
05:33:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.679359 18107 fs_manager.cc:359] Metadata directory not provided
05:33:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.679550 18107 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:33:42.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.687739 18107 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.004s
05:33:42.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.692435 18146 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:33:42.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.693526 18107 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.693804 18107 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data,/tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "397672e383b244f5899681f25c7acf35"
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:33:32 on dist-test-slave-w8hg"
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.695976 18107 fs_report.cc:383] FS layout report
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/data
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:33:42.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:33:42.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:33:42.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:33:42.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:33:42.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:33:42.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.747893 18107 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:33:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.753566 18107 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:33:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.753988 18107 kserver.cc:163] Server-wide thread pool size limit: 3276
05:33:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.894965 18107 rpc_server.cc:307] RPC server started. Bound to: 127.17.66.126:42991
05:33:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.895017 18200 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.66.126:42991 every 8 connection(s)
05:33:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.898864 18107 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/data/info.pb
05:33:42.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.900444 17673 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu as pid 18107
05:33:42.901 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:33:42.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.912328 18201 sys_catalog.cc:263] Verifying existing consensus state
05:33:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260925 05:33:42.913209 18201 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:33:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:42.914256 18107 master.cc:537] Master@127.17.66.126:42991 shutting down...
05:33:42.983 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:43.055135 18107 master.cc:552] Master@127.17.66.126:42991 shutdown complete.
05:33:43.058 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.66.124:39483(127.17.66.124:39483)] connecting to peer
05:33:43.061 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskSNHo_2/test-tmp/mini-kudu-cluster3342988048910721033/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:33:43.117 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.120 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.120 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.17.66.124:39483(127.17.66.124:39483)] Successfully connected to peer
05:33:43.121 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.121 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.122 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.189 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
05:33:43.198 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.17.66.124
05:33:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:43 dist-test-slave-w8hg krb5kdc[17685](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314416, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.66.124@KRBTEST.COM
05:33:43.413 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf8af26f4, L:/127.0.0.1:57642 - R:/127.17.66.124:39483] using SASL/GSSAPI
05:33:43.428 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:43.449 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.453 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.455 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.455 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.456 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.461 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:43.468 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.473 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.475 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:43.488 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.489 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.494 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:43.508 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.509 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.513 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.513 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.514 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.516 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:43.548 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.550 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.553 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.553 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.555 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:43.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.611 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.613 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.613 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.613 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.614 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.614 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.615 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:43.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:43.644223 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:43.645705 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:43.708 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.710 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.711 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:43.908 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:43.910 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:43.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:43.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.915 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:43.915 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:44.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:44.169 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:44.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:44.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:44.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:44.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.534 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:44.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:44.648541 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:44.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:44.648792 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:44.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:44.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:44.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:44.673 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.673 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:44.674 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:45.292796 18211 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:33:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:45.293216 18211 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:45.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:45.294710 18211 leader_election.cc:288] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991), 69a963369cde45378346599392aa251d (127.17.66.125:35429)
05:33:45.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:45.300884 18057 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991): Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111)
05:33:45.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:45.301368 18054 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 69a963369cde45378346599392aa251d (127.17.66.125:35429): Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:45.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:45.301739 18054 leader_election.cc:302] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25c80047dd1d4619acdbbf4cffffcf6a; no voters: 397672e383b244f5899681f25c7acf35, 69a963369cde45378346599392aa251d
05:33:45.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:45.302393 18211 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:33:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:45.651515 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:45.651758 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:45.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:45.658931 18031 master_runner.cc:186] Error getting master registration for 127.17.66.126:42991: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111),
05:33:45.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:45.659341 18031 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:33:46.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:46.654541 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:46.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:46.654778 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:46.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:46.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:46.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:46.971 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:46.971 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:46.973 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:46.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:46.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:46.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:47.657464 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:47.657711 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:48.660380 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:48.660622 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:49.388 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:49.390 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:49.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:49.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:49.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:49.393 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:49.393 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:49.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:49.394 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:49.499616 18221 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:33:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:49.500254 18221 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:49.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:49.502259 18221 leader_election.cc:288] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991), 69a963369cde45378346599392aa251d (127.17.66.125:35429)
05:33:49.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:49.504869 18057 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111) [suppressed 5 similar messages]
05:33:49.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:49.507700 18054 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 69a963369cde45378346599392aa251d (127.17.66.125:35429): Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:49.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:49.508162 18057 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991): Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111)
05:33:49.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:49.508534 18057 leader_election.cc:302] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25c80047dd1d4619acdbbf4cffffcf6a; no voters: 397672e383b244f5899681f25c7acf35, 69a963369cde45378346599392aa251d
05:33:49.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:49.509292 18221 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:33:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:49.663803 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:49.664104 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:33:50 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314430, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:33:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:50.383558 18049 init.cc:295] Successfully reacquired a new kerberos TGT
05:33:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:50.662319 18031 master_runner.cc:186] Error getting master registration for 127.17.66.126:42991: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111),
05:33:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:50.662704 18031 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:33:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:50.666944 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:50.667213 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:51.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:51.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:51.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:51.670869 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:51.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:51.671505 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:51.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:51.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:51.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:51.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:51.673 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:51.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:51.676 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:52.592231 17096 debug-util.cc:398] Leaking SignalData structure 0x7b0800067c40 after lost signal to thread 17031
05:33:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:52.592875 17096 debug-util.cc:398] Leaking SignalData structure 0x7b0800088d60 after lost signal to thread 17099
05:33:52.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:52.674299 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:52.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:52.674535 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:53.677498 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:53.677793 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:54.680516 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:54.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:54.680751 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:55.008 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:55.010 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:55.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:55.011 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:55.011 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:55.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:55.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:55.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:55.014 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:55.665091 18031 master_runner.cc:186] Error getting master registration for 127.17.66.126:42991: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111),
05:33:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:55.665447 18031 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:33:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:55.683506 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:55.683749 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:55.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:55.965444 18231 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:33:55.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:55.965921 18231 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:33:55.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:55.967540 18231 leader_election.cc:288] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991), 69a963369cde45378346599392aa251d (127.17.66.125:35429)
05:33:55.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:55.968621 18057 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:33:55.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:55.971769 18057 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991): Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111)
05:33:55.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:55.972239 18054 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 69a963369cde45378346599392aa251d (127.17.66.125:35429): Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:33:55.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:55.972589 18054 leader_election.cc:302] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25c80047dd1d4619acdbbf4cffffcf6a; no voters: 397672e383b244f5899681f25c7acf35, 69a963369cde45378346599392aa251d
05:33:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:33:55.973176 18231 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:33:56.249 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:56.250 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:56.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:56.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:56.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:56.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:56.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:56.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:56.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:56.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:56.686558 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:56.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:56.686800 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:57.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:57.689486 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:57.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:57.689721 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:58.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:58.692337 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:58.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:58.692572 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:33:59.268 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:33:59.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:33:59.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:59.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:59.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:59.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:59.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:33:59.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:33:59.274 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:33:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:59.695608 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:33:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:33:59.695927 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:00.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:34:00 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314440, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:34:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:00.386103 18049 init.cc:295] Successfully reacquired a new kerberos TGT
05:34:00.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:00.668323 18031 master_runner.cc:186] Error getting master registration for 127.17.66.126:42991: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111),
05:34:00.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:00.668740 18031 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:34:00.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:00.699256 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:00.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:00.699517 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:00.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:34:00.910 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:34:00.913 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:00.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:00.913 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:00.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:00.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:00.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:00.914 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:34:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:01.702518 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:01.702764 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:01.788 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:34:01.789 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:34:01.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:01.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:01.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:01.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:01.791 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:01.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:01.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:34:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:02.705521 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:02.705766 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:03.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:03.049192 18241 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:34:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:03.049639 18241 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "397672e383b244f5899681f25c7acf35" member_type: VOTER last_known_addr { host: "127.17.66.126" port: 42991 } } peers { permanent_uuid: "69a963369cde45378346599392aa251d" member_type: VOTER last_known_addr { host: "127.17.66.125" port: 35429 } } peers { permanent_uuid: "25c80047dd1d4619acdbbf4cffffcf6a" member_type: VOTER last_known_addr { host: "127.17.66.124" port: 39483 } }
05:34:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:03.051285 18241 leader_election.cc:288] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991), 69a963369cde45378346599392aa251d (127.17.66.125:35429)
05:34:03.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:03.053390 18057 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:34:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:03.056154 18057 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 397672e383b244f5899681f25c7acf35 (127.17.66.126:42991): Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111)
05:34:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:03.057237 18054 leader_election.cc:334] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 69a963369cde45378346599392aa251d (127.17.66.125:35429): Network error: Client connection negotiation failed: client connection to 127.17.66.125:35429: connect: Connection refused (error 111)
05:34:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:03.057639 18054 leader_election.cc:302] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25c80047dd1d4619acdbbf4cffffcf6a; no voters: 397672e383b244f5899681f25c7acf35, 69a963369cde45378346599392aa251d
05:34:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:03.058362 18241 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:34:03.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:34:03.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:34:03.670 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:03.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:03.670 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:03.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:03.670 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:03.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:03.672 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:34:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:03.708510 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:03.708775 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:04.711621 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:04.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:04.711902 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:05.508 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:34:05.509 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:34:05.510 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:05.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:05.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:05.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:05.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:05.511 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:05.514 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:34:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:05.671525 18031 master_runner.cc:186] Error getting master registration for 127.17.66.126:42991: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111),
05:34:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:05.671871 18031 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:34:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:05.714957 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:05.715230 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:06.718150 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:06.718436 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:07.368 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:34:07.369 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:34:07.370 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:07.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:07.370 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:07.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:07.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:07.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:07.372 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:34:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:07.721504 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:07.721771 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:07.788 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:34:07.789 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:34:07.790 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:07.790 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:07.790 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:07.790 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:07.790 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:07.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:07.792 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:34:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:08.724450 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:08.724726 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:09.468 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:34:09.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:34:09.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:09.470 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:09.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:09.470 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:09.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:09.470 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:09.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:34:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:09.727864 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:09.728223 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:34:10 dist-test-slave-w8hg krb5kdc[17685](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314450, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.66.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:34:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:10.388381 18049 init.cc:295] Successfully reacquired a new kerberos TGT
05:34:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:10.674814 18031 master_runner.cc:186] Error getting master registration for 127.17.66.126:42991: Network error: Client connection negotiation failed: client connection to 127.17.66.126:42991: connect: Connection refused (error 111),
05:34:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:10.675200 18031 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:34:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:10.731009 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:10.731266 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:11.733668 18120 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25c80047dd1d4619acdbbf4cffffcf6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:34:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:34:11.733930 18120 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:34:12.328 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.125:35429(127.17.66.125:35429)] connecting to peer
05:34:12.329 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.66.126:42991(127.17.66.126:42991)] connecting to peer
05:34:12.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:12.330 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:12.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.66.126:42991(127.17.66.126:42991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:12.330 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.125:35429(127.17.66.125:35429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:12.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.126:42991: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991
05:34:12.331 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.66.125:35429: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
05:34:12.333 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991; will retry
05:34:12.336 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29426), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [204ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [207ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [511ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [534ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [541ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [545ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [547ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [553ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [557ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [579ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [594ms] refreshing cache from master, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [597ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [600ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [602ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [634ms] refreshing cache from master, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [638ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [639ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [641ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [701ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [793ms] refreshing cache from master, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [799ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [994ms] refreshing cache from master, [994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [998ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [999ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1001ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1253ms] refreshing cache from master, [1253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1258ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1260ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1613ms] refreshing cache from master, [1614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1617ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1619ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1620ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1754ms] refreshing cache from master, [1754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1758ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1760ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1761ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [4054ms] refreshing cache from master, [4054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [4057ms] too many traces: truncated at 100 traces, deferred=Deferred@1013596433(state=PENDING, result=null, callback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325), errback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325)))
05:34:12.339 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.66.125:35429(127.17.66.125:35429)] disconnecting while in state TERMINATED
05:34:12.340 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.66.124:39483(127.17.66.124:39483)] disconnecting while in state READY
05:34:12.341 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.66.126:42991(127.17.66.126:42991)] disconnecting while in state TERMINATED
05:34:12.345 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:34:12.346 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.17.66.124:39483(127.17.66.124:39483)] handling channelInactive
05:34:12.347 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.66.124:39483(127.17.66.124:39483)] cleaning up while in state READY due to: connection closed
05:34:12.352 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:34:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:12.357523 17673 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 18031
05:34:13.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:34:13Z chronyd exiting
05:34:14.563 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:34:14.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:14.564941 16945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 17224
05:34:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:14.590161 16945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 17384
05:34:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:14.617060 16945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 17528
05:34:14.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:14.644877 16945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 16957
05:34:14.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:14.684520 16945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 17027
05:34:14.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:34:14.717892 16945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskSNHo_2/build/tsan/bin/kudu with pid 17102
05:34:15.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:34:15Z chronyd exiting
05:34:16.890 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29426), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [204ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [207ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [511ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [534ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [541ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [545ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [547ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [553ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [557ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [579ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [594ms] refreshing cache from master, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [597ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [600ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [602ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [634ms] refreshing cache from master, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [638ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [639ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [641ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [701ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [793ms] refreshing cache from master, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [799ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [994ms] refreshing cache from master, [994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [998ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [999ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1001ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1253ms] refreshing cache from master, [1253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1258ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1260ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1613ms] refreshing cache from master, [1614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1617ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1619ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1620ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1754ms] refreshing cache from master, [1754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1758ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1760ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1761ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [4054ms] refreshing cache from master, [4054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [4057ms] too many traces: truncated at 100 traces, deferred=Deferred@1013596433(state=PENDING, result=null, callback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325), errback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.66.125:35429
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29426), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [204ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [207ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [511ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [534ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [541ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [545ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [547ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [553ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [557ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [579ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [594ms] refreshing cache from master, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [597ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [600ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [602ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [634ms] refreshing cache from master, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [638ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [639ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [641ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [701ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [793ms] refreshing cache from master, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [799ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [994ms] refreshing cache from master, [994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [998ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [999ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1001ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1253ms] refreshing cache from master, [1253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1258ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1260ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1613ms] refreshing cache from master, [1614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1617ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1619ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1620ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1754ms] refreshing cache from master, [1754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1758ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1760ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1761ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [4054ms] refreshing cache from master, [4054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [4057ms] too many traces: truncated at 100 traces, deferred=Deferred@1013596433(state=PENDING, result=null, callback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325), errback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.66.125:35429
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
05:34:18.094 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
05:34:18.094 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29426), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [204ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [207ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [511ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [534ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [541ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [545ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [547ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [553ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [557ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [579ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [594ms] refreshing cache from master, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [597ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [600ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [602ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [634ms] refreshing cache from master, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [638ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [639ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [641ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [701ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [793ms] refreshing cache from master, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [799ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [994ms] refreshing cache from master, [994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [998ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [999ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1001ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1253ms] refreshing cache from master, [1253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1258ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1260ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1613ms] refreshing cache from master, [1614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1617ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1619ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1620ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1754ms] refreshing cache from master, [1754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1758ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1760ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1761ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [4054ms] refreshing cache from master, [4054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [4057ms] too many traces: truncated at 100 traces, deferred=Deferred@1013596433(state=PENDING, result=null, callback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325), errback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.66.125:35429
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29426), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [204ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [207ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [511ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [534ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [541ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [545ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [547ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [553ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [557ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [578ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [579ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [594ms] refreshing cache from master, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [597ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [600ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [602ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [634ms] refreshing cache from master, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [638ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [639ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [641ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [699ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [701ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [793ms] refreshing cache from master, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [797ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [799ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [994ms] refreshing cache from master, [994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [998ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [999ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1001ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1253ms] refreshing cache from master, [1253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1258ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1260ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1613ms] refreshing cache from master, [1614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1617ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1619ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1620ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1754ms] refreshing cache from master, [1754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [1758ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.126:42991: Network error: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.125:35429: Network error: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [1760ms] Sub RPC ConnectToMaster: received response from server master-127.17.66.124:39483: OK, [1761ms] delaying RPC due to: Service unavailable: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429, [4054ms] refreshing cache from master, [4054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.125:35429, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.124:39483, [4055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.66.126:42991, [4057ms] too many traces: truncated at 100 traces, deferred=Deferred@1013596433(state=PENDING, result=null, callback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325), errback=(continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863) -> (continuation of Deferred@2041054608 after retry RPC after error@532979513) -> (continuation of Deferred@339905999 after retry RPC after error@2104795730) -> (continuation of Deferred@1145679081 after retry RPC after error@2144329230) -> (continuation of Deferred@2137196749 after retry RPC after error@668812097) -> (continuation of Deferred@779783607 after retry RPC after error@792988120) -> (continuation of Deferred@1216764317 after retry RPC after error@93117575) -> (continuation of Deferred@2145513140 after retry RPC after error@1885614681) -> (continuation of Deferred@173328249 after retry RPC after error@1776241232) -> (continuation of Deferred@431721873 after retry RPC after error@1209827845) -> (continuation of Deferred@1692396400 after retry RPC after error@2000793713) -> (continuation of Deferred@844332045 after retry RPC after error@519298280) -> (continuation of Deferred@90972205 after retry RPC after error@2136263369) -> (continuation of Deferred@1447977530 after retry RPC after error@1448800391) -> (continuation of Deferred@852741929 after retry RPC after error@1795705144) -> (continuation of Deferred@1324530129 after retry RPC after error@1328391744) -> (continuation of Deferred@1663795723 after retry RPC after error@1254315743) -> (continuation of Deferred@287593063 after retry RPC after error@857340325)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.66.125:35429,127.17.66.124:39483,127.17.66.126:42991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.126:42991(127.17.66.126:42991): Connection refused: /127.17.66.126:42991,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.66.125:35429(127.17.66.125:35429): Connection refused: /127.17.66.125:35429
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.66.125:35429
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)