13:59:29.619 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
13:59:29.632 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task77Adji/test-tmp
13:59:29.634 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:59:29.638 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task77Adji/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:59:29.932 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
13:59:30.218 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:59:30.219 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:59:30.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.224403 24228 mini_oidc.cc:152] Starting JWKS server
13:59:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.229458 24228 webserver.cc:460] Webserver started at http://127.0.0.1:36929/ using document root <none> and password file <none>
13:59:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.230165 24228 mini_oidc.cc:189] Starting OIDC Discovery server
13:59:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.232239 24228 webserver.cc:460] Webserver started at http://127.0.0.1:40509/ using document root <none> and password file <none>
13:59:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T13:59:30Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:59:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T13:59:30Z Disabled control of system clock
13:59:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.293532 24228 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/wal
13:59:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/data
13:59:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/logs
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/data/info.pb
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.62:37917
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.62
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36929/jwks.json with env {}
13:59:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.609838 24241 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.610440 24241 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:30.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.611179 24241 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.642779 24241 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.643079 24241 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.643353 24241 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.643597 24241 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.643879 24241 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.644114 24241 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36929/jwks.json
13:59:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.681057 24241 master_runner.cc:386] Master server non-default flags:
13:59:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/data
13:59:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/wal
13:59:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.62:37917
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36929/jwks.json
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/data/info.pb
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.62
13:59:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/logs
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.683039 24241 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.685618 24241 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:30.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.713497 24251 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.713562 24247 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:30.714116 24248 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:30.715740 24241 server_base.cc:1034] running on GCE node
13:59:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.012889 24241 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:32.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.017026 24241 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:32.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.018582 24241 hybrid_clock.cc:648] HybridClock initialized: now 1786283972018534 us; error 122 us; skew 500 ppm
13:59:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.019786 24241 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:32.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.031461 24241 webserver.cc:460] Webserver started at http://127.23.169.62:41459/ using document root <none> and password file <none>
13:59:32.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.032912 24241 fs_manager.cc:359] Metadata directory not provided
13:59:32.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.033115 24241 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:32.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.033717 24241 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:32.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.039357 24241 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/data/instance:
13:59:32.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "408ab6b682b0482b93619159e4d1c039"
13:59:32.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:32 on dist-test-slave-vxj2"
13:59:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.040808 24241 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/wal/instance:
13:59:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "408ab6b682b0482b93619159e4d1c039"
13:59:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:32 on dist-test-slave-vxj2"
13:59:32.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.050176 24241 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
13:59:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.057623 24259 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.059122 24241 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
13:59:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.059593 24241 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/wal
13:59:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "408ab6b682b0482b93619159e4d1c039"
13:59:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:32 on dist-test-slave-vxj2"
13:59:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.060086 24241 fs_report.cc:383] FS layout report
13:59:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/wal
13:59:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/wal
13:59:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/data/data
13:59:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:32.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.104128 24241 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.106225 24241 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.106880 24241 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.185984 24241 rpc_server.cc:307] RPC server started. Bound to: 127.23.169.62:37917
13:59:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.186018 24313 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.169.62:37917 every 8 connection(s)
13:59:32.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.189543 24241 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/data/info.pb
13:59:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.196942 24228 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 24241
13:59:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.197474 24228 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-0/wal/instance
13:59:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.198315 24314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:59:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.201287 24228 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/wal
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/data
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/logs
13:59:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/data/info.pb
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.61:35997
13:59:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.61
13:59:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36929/jwks.json with env {}
13:59:32.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.215346 24314 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } has no permanent_uuid. Determining permanent_uuid...
13:59:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.250429 24314 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } has no permanent_uuid. Determining permanent_uuid...
13:59:32.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.252981 24263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.169.61:35997: connect: Connection refused (error 111)
13:59:32.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.257424 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.61:35997: Network error: Client connection negotiation failed: client connection to 127.23.169.61:35997: connect: Connection refused (error 111)
13:59:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.306913 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } attempt: 1
13:59:32.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.311113 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.61:35997: Network error: Client connection negotiation failed: client connection to 127.23.169.61:35997: connect: Connection refused (error 111)
13:59:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.379532 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } attempt: 2
13:59:32.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.383733 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.61:35997: Network error: Client connection negotiation failed: client connection to 127.23.169.61:35997: connect: Connection refused (error 111)
13:59:32.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.475138 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } attempt: 3
13:59:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.479138 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.61:35997: Network error: Client connection negotiation failed: client connection to 127.23.169.61:35997: connect: Connection refused (error 111)
13:59:32.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.556361 24316 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.556934 24316 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.557626 24316 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:32.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.589502 24316 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.589782 24316 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.589985 24316 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.590162 24316 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.590337 24316 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.590466 24316 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36929/jwks.json
13:59:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.622633 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } attempt: 4
13:59:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.626588 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.61:35997: Network error: Client connection negotiation failed: client connection to 127.23.169.61:35997: connect: Connection refused (error 111)
13:59:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.627220 24316 master_runner.cc:386] Master server non-default flags:
13:59:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/data
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/wal
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.61:35997
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36929/jwks.json
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/data/info.pb
13:59:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.61
13:59:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/logs
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.628548 24316 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:32.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.630781 24316 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:32.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.648922 24326 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:32.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.926075 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } attempt: 5
13:59:32.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.930740 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.61:35997: Network error: Client connection negotiation failed: client connection to 127.23.169.61:35997: connect: Connection refused (error 111)
13:59:33.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.478175 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } attempt: 6
13:59:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:33.487936 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.61:35997: Network error: Client connection negotiation failed: client connection to 127.23.169.61:35997: connect: Connection refused (error 111)
13:59:33.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.648936 24325 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:33.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:32.652050 24316 server_base.cc:1034] running on GCE node
13:59:33.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:32.649843 24328 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.877743 24316 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.880573 24316 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:33.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.881964 24316 hybrid_clock.cc:648] HybridClock initialized: now 1786283973881931 us; error 42 us; skew 500 ppm
13:59:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.883152 24316 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:33.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.890262 24316 webserver.cc:460] Webserver started at http://127.23.169.61:34367/ using document root <none> and password file <none>
13:59:33.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.891248 24316 fs_manager.cc:359] Metadata directory not provided
13:59:33.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.891453 24316 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:33.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.891867 24316 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:33.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.896782 24316 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/data/instance:
13:59:33.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1a2749223714404bb2e4c3dc031c4d0"
13:59:33.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:33 on dist-test-slave-vxj2"
13:59:33.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.897984 24316 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/wal/instance:
13:59:33.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1a2749223714404bb2e4c3dc031c4d0"
13:59:33.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:33 on dist-test-slave-vxj2"
13:59:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.905324 24316 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
13:59:33.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.911294 24337 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:33.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.912416 24316 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
13:59:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.912801 24316 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/wal
13:59:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1a2749223714404bb2e4c3dc031c4d0"
13:59:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:33 on dist-test-slave-vxj2"
13:59:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.913237 24316 fs_report.cc:383] FS layout report
13:59:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/wal
13:59:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/wal
13:59:33.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/data/data
13:59:33.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:33.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:33.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:33.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.955644 24316 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:33.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.957305 24316 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:33.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:33.957856 24316 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:34.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.034389 24316 rpc_server.cc:307] RPC server started. Bound to: 127.23.169.61:35997
13:59:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.034504 24392 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.169.61:35997 every 8 connection(s)
13:59:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.037348 24316 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/data/info.pb
13:59:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.040126 24228 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 24316
13:59:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.040699 24228 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-1/wal/instance
13:59:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.042865 24228 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/wal
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/data
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/logs
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/data/info.pb
13:59:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.60:35187
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.60
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:34.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36929/jwks.json with env {}
13:59:34.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.043181 24393 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:59:34.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.055510 24393 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } has no permanent_uuid. Determining permanent_uuid...
13:59:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.079535 24393 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } has no permanent_uuid. Determining permanent_uuid...
13:59:34.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.097232 24393 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } has no permanent_uuid. Determining permanent_uuid...
13:59:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.099624 24339 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.104575 24393 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.154225 24393 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 1
13:59:34.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.159080 24393 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.227541 24393 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 2
13:59:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.231964 24393 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.323385 24393 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 3
13:59:34.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.327427 24393 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.387915 24395 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:34.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.388800 24395 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:34.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.389998 24395 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:34.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.434803 24395 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:34.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.435241 24395 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:34.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.435456 24395 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:34.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.435710 24395 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:34.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.435917 24395 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:34.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.436066 24395 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36929/jwks.json
13:59:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.470818 24393 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 4
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.474660 24393 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.474956 24395 master_runner.cc:386] Master server non-default flags:
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/data
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/wal
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.60:35187
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36929/jwks.json
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/data/info.pb
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.60
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/logs
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.476223 24395 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.478557 24395 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:34.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.498476 24407 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:34.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.548399 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } attempt: 7
13:59:34.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.565713 24314 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } has no permanent_uuid. Determining permanent_uuid...
13:59:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.571453 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.629885 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 1
13:59:34.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.635290 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.716717 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 2
13:59:34.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.721596 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.774166 24393 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 5
13:59:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.778748 24393 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.807049 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 3
13:59:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.811820 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:34.952262 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 4
13:59:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.957037 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.240536 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 5
13:59:35.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:35.245380 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:35.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.326234 24393 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 6
13:59:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:35.333575 24393 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.498523 24405 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:34.500290 24409 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:35.719544 24408 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:59:35.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.719590 24395 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:35.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.723944 24395 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.726862 24395 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.728214 24395 hybrid_clock.cc:648] HybridClock initialized: now 1786283975728180 us; error 55 us; skew 500 ppm
13:59:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.729008 24395 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.736127 24395 webserver.cc:460] Webserver started at http://127.23.169.60:33869/ using document root <none> and password file <none>
13:59:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.737020 24395 fs_manager.cc:359] Metadata directory not provided
13:59:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.737267 24395 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:35.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.737752 24395 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.745039 24395 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/data/instance:
13:59:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25b61aa1960c454da7cc2cf40ca44277"
13:59:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:35 on dist-test-slave-vxj2"
13:59:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.746634 24395 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/wal/instance:
13:59:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25b61aa1960c454da7cc2cf40ca44277"
13:59:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:35 on dist-test-slave-vxj2"
13:59:35.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.755358 24395 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.002s sys 0.007s
13:59:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.761440 24422 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.762565 24395 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
13:59:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.762944 24395 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/wal
13:59:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25b61aa1960c454da7cc2cf40ca44277"
13:59:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:35 on dist-test-slave-vxj2"
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.763342 24395 fs_report.cc:383] FS layout report
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/wal
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/wal
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/data/data
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.797796 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 6
13:59:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:35.804418 24314 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.169.60:35187: Network error: Client connection negotiation failed: client connection to 127.23.169.60:35187: connect: Connection refused (error 111)
13:59:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.808895 24395 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:35.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.810469 24395 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.810902 24395 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.893357 24395 rpc_server.cc:307] RPC server started. Bound to: 127.23.169.60:35187
13:59:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.893536 24476 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.169.60:35187 every 8 connection(s)
13:59:35.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.895960 24395 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/data/info.pb
13:59:35.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.901649 24228 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 24395
13:59:35.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.901859 24477 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:59:35.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.902069 24228 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/master-2/wal/instance
13:59:35.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.921396 24477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } has no permanent_uuid. Determining permanent_uuid...
13:59:35.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.950847 24477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } has no permanent_uuid. Determining permanent_uuid...
13:59:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.962801 24477 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } has no permanent_uuid. Determining permanent_uuid...
13:59:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.980547 24477 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277: Bootstrap starting.
13:59:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.989475 24477 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277: Neither blocks nor log segments found. Creating new log.
13:59:35.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.992581 24477 log.cc:824] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277: Log is configured to *not* fsync() on all Append() calls
13:59:35.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:35.998647 24477 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277: No bootstrap required, opened a new log
13:59:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.025655 24477 raft_consensus.cc:348] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.026365 24477 raft_consensus.cc:374] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:59:36.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.026692 24477 raft_consensus.cc:729] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 25b61aa1960c454da7cc2cf40ca44277, State: Initialized, Role: FOLLOWER
13:59:36.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.027750 24477 consensus_queue.cc:260] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.030599 24486 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.031477 24486 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: This master's current role is: FOLLOWER
13:59:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.032603 24477 sys_catalog.cc:564] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: configured and running, proceeding with master startup.
13:59:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:36.067786 24430 tablet.cc:2369] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:59:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:36.068861 24497 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:36.069211 24497 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:36.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.063546 24395 master_runner.cc:186] Error getting master registration for 127.23.169.62:37917: OK, instance_id { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" instance_seqno: 1786283972164026 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:59:36.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.092043 24395 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:59:36.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.394200 24393 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 7
13:59:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.414083 24393 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: Bootstrap starting.
13:59:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.419775 24393 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: Neither blocks nor log segments found. Creating new log.
13:59:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.421825 24393 log.cc:824] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: Log is configured to *not* fsync() on all Append() calls
13:59:36.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.426752 24393 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: No bootstrap required, opened a new log
13:59:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.450057 24393 raft_consensus.cc:348] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.450740 24393 raft_consensus.cc:374] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:59:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.451011 24393 raft_consensus.cc:729] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f1a2749223714404bb2e4c3dc031c4d0, State: Initialized, Role: FOLLOWER
13:59:36.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.451964 24393 consensus_queue.cc:260] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.454918 24502 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:36.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.455803 24502 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [sys.catalog]: This master's current role is: FOLLOWER
13:59:36.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.456748 24393 sys_catalog.cc:564] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [sys.catalog]: configured and running, proceeding with master startup.
13:59:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:36.465597 24344 tablet.cc:2369] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:59:36.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.467741 24486 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:59:36.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.468338 24486 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:36.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.476851 24366 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25b61aa1960c454da7cc2cf40ca44277" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f1a2749223714404bb2e4c3dc031c4d0" is_pre_election: true
13:59:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.477998 24366 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 25b61aa1960c454da7cc2cf40ca44277 in term 0.
13:59:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.479949 24426 leader_election.cc:302] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 25b61aa1960c454da7cc2cf40ca44277, f1a2749223714404bb2e4c3dc031c4d0; no voters:
13:59:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.482363 24486 leader_election.cc:288] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 408ab6b682b0482b93619159e4d1c039 (127.23.169.62:37917), f1a2749223714404bb2e4c3dc031c4d0 (127.23.169.61:35997)
13:59:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.483848 24486 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 0 FOLLOWER]: Leader pre-election won for term 1
13:59:36.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.485435 24486 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:59:36.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.486267 24486 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 0 FOLLOWER]: Advancing to term 1
13:59:36.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.496353 24486 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.499555 24486 leader_election.cc:288] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [CANDIDATE]: Term 1 election: Requested vote from peers 408ab6b682b0482b93619159e4d1c039 (127.23.169.62:37917), f1a2749223714404bb2e4c3dc031c4d0 (127.23.169.61:35997)
13:59:36.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.500897 24366 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25b61aa1960c454da7cc2cf40ca44277" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f1a2749223714404bb2e4c3dc031c4d0"
13:59:36.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.501529 24366 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [term 0 FOLLOWER]: Advancing to term 1
13:59:36.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.508742 24366 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 25b61aa1960c454da7cc2cf40ca44277 in term 1.
13:59:36.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.510288 24426 leader_election.cc:302] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 25b61aa1960c454da7cc2cf40ca44277, f1a2749223714404bb2e4c3dc031c4d0; no voters:
13:59:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.511245 24486 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 1 FOLLOWER]: Leader election won for term 1
13:59:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.511715 24486 raft_consensus.cc:686] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [term 1 LEADER]: Becoming Leader. State: Replica: 25b61aa1960c454da7cc2cf40ca44277, State: Running, Role: LEADER
13:59:36.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.512781 24486 consensus_queue.cc:237] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.541419 24508 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25b61aa1960c454da7cc2cf40ca44277. Latest consensus state: current_term: 1 leader_uuid: "25b61aa1960c454da7cc2cf40ca44277" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.542631 24508 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: This master's current role is: LEADER
13:59:36.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.546538 24518 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:59:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:36.548830 24514 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:36.549431 24514 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:36.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.554849 24518 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:59:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.591480 24366 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [term 1 FOLLOWER]: Refusing update from remote peer 25b61aa1960c454da7cc2cf40ca44277: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:59:36.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.594122 24508 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:59:36.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.538893 24316 master_runner.cc:186] Error getting master registration for 127.23.169.62:37917: OK, instance_id { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" instance_seqno: 1786283972164026 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:59:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.619309 24316 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:59:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.647351 24519 mvcc.cc:205] Tried to move back new op lower bound from 7316619168053612544 to 7316619167811330048. Current Snapshot: MvccSnapshot[applied={T|T < 7316619168053612544}]
13:59:36.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.652094 24502 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25b61aa1960c454da7cc2cf40ca44277. Latest consensus state: current_term: 1 leader_uuid: "25b61aa1960c454da7cc2cf40ca44277" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:36.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.653707 24502 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [sys.catalog]: This master's current role is: FOLLOWER
13:59:36.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.659303 24486 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "25b61aa1960c454da7cc2cf40ca44277" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:36.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.660123 24486 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: This master's current role is: LEADER
13:59:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.664779 24518 catalog_manager.cc:1347] Generated new cluster ID: 3fd231dbe0c14eebaf488d659cdc839f
13:59:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.665089 24518 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:59:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.668291 24502 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "25b61aa1960c454da7cc2cf40ca44277" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:36.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.669215 24502 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0 [sys.catalog]: This master's current role is: FOLLOWER
13:59:36.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.723199 24518 catalog_manager.cc:1370] Generated new certificate authority record
13:59:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.730129 24518 catalog_manager.cc:1504] Loading token signing keys...
13:59:36.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.758483 24518 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277: Generated new TSK 0
13:59:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.760054 24518 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.801507 24228 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/wal
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/data
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/logs
13:59:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/data/info.pb
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.1:0
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.169.1
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.1
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:59:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.837819 24314 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } attempt: 7
13:59:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.898226 24314 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039: Bootstrap starting.
13:59:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.916332 24314 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039: Neither blocks nor log segments found. Creating new log.
13:59:36.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.922684 24314 log.cc:824] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039: Log is configured to *not* fsync() on all Append() calls
13:59:36.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:36.946297 24314 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039: No bootstrap required, opened a new log
13:59:37.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.046538 24314 raft_consensus.cc:348] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:37.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.049423 24314 raft_consensus.cc:374] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:59:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.050567 24314 raft_consensus.cc:729] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 408ab6b682b0482b93619159e4d1c039, State: Initialized, Role: FOLLOWER
13:59:37.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.053617 24314 consensus_queue.cc:260] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } }
13:59:37.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.083036 24286 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [term 0 FOLLOWER]: Advancing to term 1
13:59:37.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.085420 24287 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25b61aa1960c454da7cc2cf40ca44277" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "408ab6b682b0482b93619159e4d1c039"
13:59:37.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.087049 24288 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "25b61aa1960c454da7cc2cf40ca44277" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "408ab6b682b0482b93619159e4d1c039" is_pre_election: true
13:59:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.096160 24286 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [term 1 FOLLOWER]: Refusing update from remote peer 25b61aa1960c454da7cc2cf40ca44277: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:59:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.098256 24486 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [LEADER]: Connected to new peer: Peer: permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:59:37.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.081835 24528 sys_catalog.cc:455] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:37.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.108726 24528 sys_catalog.cc:458] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [sys.catalog]: This master's current role is: FOLLOWER
13:59:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.107266 24314 sys_catalog.cc:564] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [sys.catalog]: configured and running, proceeding with master startup.
13:59:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:37.118530 24265 tablet.cc:2369] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:59:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.215492 24528 sys_catalog.cc:455] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 25b61aa1960c454da7cc2cf40ca44277. Latest consensus state: current_term: 1 leader_uuid: "25b61aa1960c454da7cc2cf40ca44277" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.215658 24529 sys_catalog.cc:455] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "25b61aa1960c454da7cc2cf40ca44277" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.216478 24529 sys_catalog.cc:458] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [sys.catalog]: This master's current role is: FOLLOWER
13:59:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.216467 24528 sys_catalog.cc:458] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039 [sys.catalog]: This master's current role is: FOLLOWER
13:59:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.276924 24544 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
13:59:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:37.379251 24525 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:37.379820 24525 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:37.381285 24525 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:37.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:37.417847 24525 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:37.418983 24525 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.169.1
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.455785 24525 tablet_server_runner.cc:78] Tablet server non-default flags:
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/data
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/wal
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.1:0
13:59:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/data/info.pb
13:59:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.1
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.169.1
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/logs
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.457518 24525 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.459741 24525 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:37.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:37.491091 24551 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:37.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.555523 24514 catalog_manager.cc:1259] Loaded cluster ID: 3fd231dbe0c14eebaf488d659cdc839f
13:59:37.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.556221 24514 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: loading cluster ID for follower catalog manager: success
13:59:37.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.564572 24514 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: acquiring CA information for follower catalog manager: success
13:59:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.572899 24514 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f1a2749223714404bb2e4c3dc031c4d0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:59:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.603672 24486 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "25b61aa1960c454da7cc2cf40ca44277" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "408ab6b682b0482b93619159e4d1c039" member_type: VOTER last_known_addr { host: "127.23.169.62" port: 37917 } } peers { permanent_uuid: "f1a2749223714404bb2e4c3dc031c4d0" member_type: VOTER last_known_addr { host: "127.23.169.61" port: 35997 } } peers { permanent_uuid: "25b61aa1960c454da7cc2cf40ca44277" member_type: VOTER last_known_addr { host: "127.23.169.60" port: 35187 } } }
13:59:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:37.604955 24486 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 [sys.catalog]: This master's current role is: LEADER
13:59:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:38.307044 24544 catalog_manager.cc:1259] Loaded cluster ID: 3fd231dbe0c14eebaf488d659cdc839f
13:59:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:38.307452 24544 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039: loading cluster ID for follower catalog manager: success
13:59:38.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:38.360903 24544 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039: acquiring CA information for follower catalog manager: success
13:59:38.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:38.401137 24544 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 408ab6b682b0482b93619159e4d1c039: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:59:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:39.422260 24553 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1930 milliseconds
13:59:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:39.422546 24554 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:38.880858 24550 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 24525
13:59:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:37.493290 24552 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:39.422794 24525 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.934s user 0.439s sys 0.704s
13:59:39.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:39.425029 24525 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.936s user 0.439s sys 0.704s
13:59:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.425587 24525 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.429775 24525 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.453778 24525 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:39.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.455669 24525 hybrid_clock.cc:648] HybridClock initialized: now 1786283979455621 us; error 54 us; skew 500 ppm
13:59:39.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.457358 24525 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.496833 24525 webserver.cc:460] Webserver started at http://127.23.169.1:45927/ using document root <none> and password file <none>
13:59:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.498620 24525 fs_manager.cc:359] Metadata directory not provided
13:59:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.499187 24525 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.499881 24525 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.507431 24525 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/data/instance:
13:59:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf4d1820973649d2b1739d47cfc0352e"
13:59:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:39 on dist-test-slave-vxj2"
13:59:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.509476 24525 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/wal/instance:
13:59:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf4d1820973649d2b1739d47cfc0352e"
13:59:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:39 on dist-test-slave-vxj2"
13:59:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.531538 24525 fs_manager.cc:691] Time spent creating directory manager: real 0.021s user 0.017s sys 0.001s
13:59:39.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.546437 24564 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.547956 24525 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.008s sys 0.000s
13:59:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.548393 24525 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/wal
13:59:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf4d1820973649d2b1739d47cfc0352e"
13:59:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:39 on dist-test-slave-vxj2"
13:59:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.548856 24525 fs_report.cc:383] FS layout report
13:59:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/wal
13:59:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/wal
13:59:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/data/data
13:59:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:39.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.634760 24525 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:39.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.637084 24525 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:39.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.637707 24525 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:39.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.647163 24525 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:59:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.660355 24525 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:59:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.660764 24525 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:59:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.661249 24525 ts_tablet_manager.cc:610] Registered 0 tablets
13:59:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:39.661492 24525 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:59:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.094372 24525 rpc_server.cc:307] RPC server started. Bound to: 127.23.169.1:35863
13:59:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.094519 24682 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.169.1:35863 every 8 connection(s)
13:59:40.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.097116 24525 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/data/info.pb
13:59:40.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.098760 24228 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 24525
13:59:40.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.099303 24228 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-0/wal/instance
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.118651 24228 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/wal
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/data
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/logs
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/data/info.pb
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.2:0
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.169.2
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.2
13:59:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:59:40.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.187455 24683 heartbeater.cc:347] Connected to a master server at 127.23.169.60:35187
13:59:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.188020 24683 heartbeater.cc:464] Registering TS with master...
13:59:40.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.189692 24683 heartbeater.cc:511] Master 127.23.169.60:35187 requested a full tablet report, sending...
13:59:40.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.191335 24686 heartbeater.cc:347] Connected to a master server at 127.23.169.62:37917
13:59:40.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.191663 24686 heartbeater.cc:464] Registering TS with master...
13:59:40.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.192495 24686 heartbeater.cc:511] Master 127.23.169.62:37917 requested a full tablet report, sending...
13:59:40.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.193971 24684 heartbeater.cc:347] Connected to a master server at 127.23.169.61:35997
13:59:40.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.194331 24684 heartbeater.cc:464] Registering TS with master...
13:59:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.195245 24684 heartbeater.cc:511] Master 127.23.169.61:35997 requested a full tablet report, sending...
13:59:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.196733 24277 ts_manager.cc:194] Registered new tserver with Master: bf4d1820973649d2b1739d47cfc0352e (127.23.169.1:35863)
13:59:40.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.198257 24441 ts_manager.cc:194] Registered new tserver with Master: bf4d1820973649d2b1739d47cfc0352e (127.23.169.1:35863)
13:59:40.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.201831 24441 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.169.1:46917
13:59:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.205475 24356 ts_manager.cc:194] Registered new tserver with Master: bf4d1820973649d2b1739d47cfc0352e (127.23.169.1:35863)
13:59:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:40.523608 24690 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:40.524216 24690 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:40.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:40.524910 24690 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:40.558430 24690 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:40.559446 24690 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.169.2
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.600600 24690 tablet_server_runner.cc:78] Tablet server non-default flags:
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/data
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/wal
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.2:0
13:59:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/data/info.pb
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.2
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.169.2
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/logs
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.602322 24690 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:40.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:40.604599 24690 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:40.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:40.627125 24705 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:41.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.210457 24683 heartbeater.cc:503] Master 127.23.169.60:35187 was elected leader, sending a full tablet report...
13:59:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:40.627336 24703 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:40.628556 24702 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:41.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:41.859548 24704 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:59:41.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.859673 24690 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:41.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.863693 24690 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:41.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.866133 24690 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:41.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.867537 24690 hybrid_clock.cc:648] HybridClock initialized: now 1786283981867498 us; error 48 us; skew 500 ppm
13:59:41.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.868396 24690 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:41.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.875275 24690 webserver.cc:460] Webserver started at http://127.23.169.2:34727/ using document root <none> and password file <none>
13:59:41.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.876259 24690 fs_manager.cc:359] Metadata directory not provided
13:59:41.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.876534 24690 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:41.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.876955 24690 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:41.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.882684 24690 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/data/instance:
13:59:41.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c8c9a7cf3a24edabf5d83d2c7f9ef46"
13:59:41.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:41 on dist-test-slave-vxj2"
13:59:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.884109 24690 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/wal/instance:
13:59:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c8c9a7cf3a24edabf5d83d2c7f9ef46"
13:59:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:41 on dist-test-slave-vxj2"
13:59:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.892572 24690 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.005s
13:59:41.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.898893 24714 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.900058 24690 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
13:59:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.900461 24690 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/wal
13:59:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c8c9a7cf3a24edabf5d83d2c7f9ef46"
13:59:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:41 on dist-test-slave-vxj2"
13:59:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.900847 24690 fs_report.cc:383] FS layout report
13:59:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/wal
13:59:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/wal
13:59:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/data/data
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:41.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.942307 24690 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:41.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.944092 24690 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:41.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.944694 24690 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:41.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.947729 24690 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:59:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.952160 24690 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:59:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.952416 24690 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:59:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.952751 24690 ts_tablet_manager.cc:610] Registered 0 tablets
13:59:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:41.952951 24690 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:59:42.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.124468 24690 rpc_server.cc:307] RPC server started. Bound to: 127.23.169.2:44657
13:59:42.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.124584 24830 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.169.2:44657 every 8 connection(s)
13:59:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.127195 24690 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/data/info.pb
13:59:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.138208 24228 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 24690
13:59:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.139083 24228 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-1/wal/instance
13:59:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.156087 24228 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/wal
13:59:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/data
13:59:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/logs
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/data/info.pb
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.3:0
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.169.3
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.3
13:59:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:59:42.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.184660 24834 heartbeater.cc:347] Connected to a master server at 127.23.169.62:37917
13:59:42.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.185199 24834 heartbeater.cc:464] Registering TS with master...
13:59:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.186425 24834 heartbeater.cc:511] Master 127.23.169.62:37917 requested a full tablet report, sending...
13:59:42.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.189225 24833 heartbeater.cc:347] Connected to a master server at 127.23.169.61:35997
13:59:42.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.189610 24833 heartbeater.cc:464] Registering TS with master...
13:59:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.190436 24833 heartbeater.cc:511] Master 127.23.169.61:35997 requested a full tablet report, sending...
13:59:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.192461 24277 ts_manager.cc:194] Registered new tserver with Master: 1c8c9a7cf3a24edabf5d83d2c7f9ef46 (127.23.169.2:44657)
13:59:42.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.194759 24356 ts_manager.cc:194] Registered new tserver with Master: 1c8c9a7cf3a24edabf5d83d2c7f9ef46 (127.23.169.2:44657)
13:59:42.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.198630 24832 heartbeater.cc:347] Connected to a master server at 127.23.169.60:35187
13:59:42.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.198992 24832 heartbeater.cc:464] Registering TS with master...
13:59:42.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.199797 24832 heartbeater.cc:511] Master 127.23.169.60:35187 requested a full tablet report, sending...
13:59:42.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.202127 24441 ts_manager.cc:194] Registered new tserver with Master: 1c8c9a7cf3a24edabf5d83d2c7f9ef46 (127.23.169.2:44657)
13:59:42.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.204119 24441 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.169.2:35095
13:59:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:42.494761 24842 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:42.495270 24842 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:42.495698 24842 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:42.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:42.528231 24842 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:42.529069 24842 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.169.3
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.567584 24842 tablet_server_runner.cc:78] Tablet server non-default flags:
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.169.20:37055
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/data
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/wal
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.169.3:0
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/data/info.pb
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.169.3
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.169.62:37917,127.23.169.61:35997,127.23.169.60:35187
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.169.3
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/logs
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:42.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:42.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:42.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.568957 24842 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:42.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:42.571149 24842 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:42.594465 24850 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:43.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.208689 24832 heartbeater.cc:503] Master 127.23.169.60:35187 was elected leader, sending a full tablet report...
13:59:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:42.595093 24849 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:43.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:42.594568 24852 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:43.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:43.890607 24851 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1294 milliseconds
13:59:43.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.890731 24842 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:43.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.892096 24842 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.895015 24842 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.896482 24842 hybrid_clock.cc:648] HybridClock initialized: now 1786283983896417 us; error 86 us; skew 500 ppm
13:59:43.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.897356 24842 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:43.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.904294 24842 webserver.cc:460] Webserver started at http://127.23.169.3:43391/ using document root <none> and password file <none>
13:59:43.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.905233 24842 fs_manager.cc:359] Metadata directory not provided
13:59:43.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.905431 24842 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:43.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.905822 24842 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.911087 24842 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/data/instance:
13:59:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37c853403d9949aa923c2f2626f0d049"
13:59:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:43 on dist-test-slave-vxj2"
13:59:43.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.912391 24842 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/wal/instance:
13:59:43.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37c853403d9949aa923c2f2626f0d049"
13:59:43.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:43 on dist-test-slave-vxj2"
13:59:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.920017 24842 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
13:59:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.928243 24859 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.929440 24842 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
13:59:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.929869 24842 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/wal
13:59:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37c853403d9949aa923c2f2626f0d049"
13:59:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:43 on dist-test-slave-vxj2"
13:59:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.930260 24842 fs_report.cc:383] FS layout report
13:59:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/wal
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/wal
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/data/data
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.971226 24842 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.972833 24842 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:43.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.973418 24842 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:43.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.976228 24842 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:59:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.981293 24842 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:59:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.981567 24842 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:59:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.981894 24842 ts_tablet_manager.cc:610] Registered 0 tablets
13:59:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:43.982106 24842 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:59:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.151417 24842 rpc_server.cc:307] RPC server started. Bound to: 127.23.169.3:38617
13:59:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.151582 24977 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.169.3:38617 every 8 connection(s)
13:59:44.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.154350 24842 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/data/info.pb
13:59:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.155758 24228 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 24842
13:59:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.156334 24228 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster1434283733770716886/ts-2/wal/instance
13:59:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.210904 24980 heartbeater.cc:347] Connected to a master server at 127.23.169.61:35997
13:59:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.211416 24980 heartbeater.cc:464] Registering TS with master...
13:59:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.212795 24980 heartbeater.cc:511] Master 127.23.169.61:35997 requested a full tablet report, sending...
13:59:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.217646 24356 ts_manager.cc:194] Registered new tserver with Master: 37c853403d9949aa923c2f2626f0d049 (127.23.169.3:38617)
13:59:44.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.220904 24979 heartbeater.cc:347] Connected to a master server at 127.23.169.60:35187
13:59:44.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.221354 24979 heartbeater.cc:464] Registering TS with master...
13:59:44.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.222223 24979 heartbeater.cc:511] Master 127.23.169.60:35187 requested a full tablet report, sending...
13:59:44.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.225739 24441 ts_manager.cc:194] Registered new tserver with Master: 37c853403d9949aa923c2f2626f0d049 (127.23.169.3:38617)
13:59:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.227260 24981 heartbeater.cc:347] Connected to a master server at 127.23.169.62:37917
13:59:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.227603 24981 heartbeater.cc:464] Registering TS with master...
13:59:44.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.228274 24441 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.169.3:37615
13:59:44.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.228523 24981 heartbeater.cc:511] Master 127.23.169.62:37917 requested a full tablet report, sending...
13:59:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.231407 24277 ts_manager.cc:194] Registered new tserver with Master: 37c853403d9949aa923c2f2626f0d049 (127.23.169.3:38617)
13:59:44.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.236987 24228 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
13:59:44.248 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:59:44.250 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
13:59:44.260 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.169.62"
port: 37917
}
bound_http_address {
host: "127.23.169.62"
port: 41459
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.169.61"
port: 35997
}
bound_http_address {
host: "127.23.169.61"
port: 34367
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.169.60"
port: 35187
}
bound_http_address {
host: "127.23.169.60"
port: 33869
}
}
}
13:59:44.265 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:59:44.268 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.169.1"
port: 35863
}
bound_http_address {
host: "127.23.169.1"
port: 45927
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.169.2"
port: 44657
}
bound_http_address {
host: "127.23.169.2"
port: 34727
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.169.3"
port: 38617
}
bound_http_address {
host: "127.23.169.3"
port: 43391
}
}
}
13:59:44.268 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
13:59:44.489 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
13:59:44.610 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task77Adji/test-tmp
13:59:44.611 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:59:44.611 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task77Adji/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:59:44.615 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
13:59:44.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:44.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:59:44.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.966765 24992 mini_oidc.cc:152] Starting JWKS server
13:59:44.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.970403 24992 webserver.cc:460] Webserver started at http://127.0.0.1:38575/ using document root <none> and password file <none>
13:59:44.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.970928 24992 mini_oidc.cc:189] Starting OIDC Discovery server
13:59:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:44.972779 24992 webserver.cc:460] Webserver started at http://127.0.0.1:43715/ using document root <none> and password file <none>
13:59:45.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
13:59:45.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/principal' for realm 'KRBTEST.COM',
13:59:45.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
13:59:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:45 dist-test-slave-vxj2 krb5kdc[25005](info): setting up network...
13:59:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
13:59:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:45 dist-test-slave-vxj2 krb5kdc[25005](info): set up 2 sockets
13:59:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:45 dist-test-slave-vxj2 krb5kdc[25005](info): commencing operation
13:59:45.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
13:59:45.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:45.232832 24979 heartbeater.cc:503] Master 127.23.169.60:35187 was elected leader, sending a full tablet report...
13:59:46.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.211618 24992 mini_kdc.cc:121] Time spent starting KDC: real 1.212s user 0.000s sys 0.024s
13:59:46.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
13:59:46.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:59:46.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
13:59:46.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
13:59:46.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:59:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
13:59:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
13:59:46.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:59:46.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
13:59:46.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:59:46.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/test-user.keytab.
13:59:46.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/test-user.keytab.
13:59:46.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:46 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283986, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:46.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:59:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T13:59:46Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:59:46.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T13:59:46Z Disabled control of system clock
13:59:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.104.62@KRBTEST.COM; defaulting to no policy
13:59:46.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:59:46.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.104.62@KRBTEST.COM" created.
13:59:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:59:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.104.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.62.keytab.
13:59:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.104.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.62.keytab.
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:46.434940 24992 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/logs
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/info.pb
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.62:33403
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.62
13:59:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.62.keytab
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.62
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:59:46.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.755218 25019 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:46.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.755945 25019 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.756875 25019 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.790294 25019 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:59:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.790596 25019 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.791002 25019 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.791338 25019 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.791677 25019 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:46.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.792006 25019 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:46.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.792251 25019 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38575/jwks.json
13:59:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:46.829241 25019 master_runner.cc:386] Master server non-default flags:
13:59:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.62.keytab
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.62
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.62:33403
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/info.pb
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.62
13:59:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/logs
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:46.830821 25019 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:46.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:46.832199 25019 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:46.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.851522 25025 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.852751 25026 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:46.854497 25019 server_base.cc:1034] running on GCE node
13:59:48.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:46.851761 25028 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:48.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.157011 25019 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:48.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.160584 25019 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.162194 25019 hybrid_clock.cc:648] HybridClock initialized: now 1786283988162143 us; error 115 us; skew 500 ppm
13:59:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.163255 25019 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:48.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:48 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283988, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:48.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.169832 25019 init.cc:377] Logged in from keytab as kudu/127.24.104.62@KRBTEST.COM (short username kudu)
13:59:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.175050 25019 webserver.cc:460] Webserver started at http://127.24.104.62:39629/ using document root <none> and password file <none>
13:59:48.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.176045 25019 fs_manager.cc:359] Metadata directory not provided
13:59:48.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.176306 25019 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:48.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.176800 25019 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.182058 25019 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/instance:
13:59:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc708307afc847cfb7289b0d0d8ef187"
13:59:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:48 on dist-test-slave-vxj2"
13:59:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.183266 25019 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal/instance:
13:59:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc708307afc847cfb7289b0d0d8ef187"
13:59:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:48 on dist-test-slave-vxj2"
13:59:48.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.191202 25019 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
13:59:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.197746 25038 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.198890 25019 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
13:59:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.199285 25019 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc708307afc847cfb7289b0d0d8ef187"
13:59:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:48 on dist-test-slave-vxj2"
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.199630 25019 fs_report.cc:383] FS layout report
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/data
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.266847 25019 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.273399 25019 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:48.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.273980 25019 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.352990 25019 rpc_server.cc:307] RPC server started. Bound to: 127.24.104.62:33403
13:59:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.353080 25091 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.104.62:33403 every 8 connection(s)
13:59:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.356022 25019 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/info.pb
13:59:48.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.361995 25092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:59:48.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.362689 24992 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 25019
13:59:48.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.363111 24992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal/instance
13:59:48.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.376487 25092 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } has no permanent_uuid. Determining permanent_uuid...
13:59:48.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.104.61@KRBTEST.COM; defaulting to no policy
13:59:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:59:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.104.61@KRBTEST.COM" created.
13:59:48.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:48 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283988, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.62@KRBTEST.COM for kudu/127.24.104.62@KRBTEST.COM
13:59:48.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:59:48.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.104.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.61.keytab.
13:59:48.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.104.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.61.keytab.
13:59:48.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.418776 24992 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:48.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:48.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/logs
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/info.pb
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.61:35389
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.61
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.61.keytab
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.61
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:59:48.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.424090 25092 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } has no permanent_uuid. Determining permanent_uuid...
13:59:48.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.426991 25041 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:48.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.431635 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.61:35389: Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.481192 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } attempt: 1
13:59:48.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.485229 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.61:35389: Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:48.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.553642 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } attempt: 2
13:59:48.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.557550 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.61:35389: Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:48.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.648940 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } attempt: 3
13:59:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.652885 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.61:35389: Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:48.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.746302 25098 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.746896 25098 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.747495 25098 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.779701 25098 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:59:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.780002 25098 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.780452 25098 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.780807 25098 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.781118 25098 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.781399 25098 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.781626 25098 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38575/jwks.json
13:59:48.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.796380 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } attempt: 4
13:59:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.800081 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.61:35389: Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.819684 25098 master_runner.cc:386] Master server non-default flags:
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.61.keytab
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.61
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.61:35389
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:59:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/info.pb
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.61
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/logs
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.820951 25098 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.822211 25098 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.842170 25106 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:49.099575 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } attempt: 5
13:59:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:49.103567 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.61:35389: Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:49.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:49.651145 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } attempt: 6
13:59:49.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:49.659076 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.61:35389: Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:50.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.842173 25107 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:50.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:48.844959 25098 server_base.cc:1034] running on GCE node
13:59:50.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:48.843526 25109 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:50.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.091322 25098 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:50.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.094592 25098 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.096099 25098 hybrid_clock.cc:648] HybridClock initialized: now 1786283990096036 us; error 81 us; skew 500 ppm
13:59:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.097209 25098 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:50 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283990, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.109305 25098 init.cc:377] Logged in from keytab as kudu/127.24.104.61@KRBTEST.COM (short username kudu)
13:59:50.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.115206 25098 webserver.cc:460] Webserver started at http://127.24.104.61:37399/ using document root <none> and password file <none>
13:59:50.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.116629 25098 fs_manager.cc:359] Metadata directory not provided
13:59:50.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.116878 25098 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:50.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.117451 25098 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.122789 25098 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/instance:
13:59:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ba3be881dcf4672990844c9c655cc60"
13:59:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:50 on dist-test-slave-vxj2"
13:59:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.123991 25098 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal/instance:
13:59:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ba3be881dcf4672990844c9c655cc60"
13:59:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:50 on dist-test-slave-vxj2"
13:59:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.131598 25098 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
13:59:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.137943 25119 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.139055 25098 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
13:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.139429 25098 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ba3be881dcf4672990844c9c655cc60"
13:59:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:50 on dist-test-slave-vxj2"
13:59:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.139778 25098 fs_report.cc:383] FS layout report
13:59:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/data
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.179072 25098 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:50.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.185784 25098 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:50.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.186599 25098 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.278398 25098 rpc_server.cc:307] RPC server started. Bound to: 127.24.104.61:35389
13:59:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.278472 25179 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.104.61:35389 every 8 connection(s)
13:59:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.281345 25098 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/info.pb
13:59:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.287616 25180 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:59:50.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.289528 24992 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 25098
13:59:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.290019 24992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal/instance
13:59:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.300509 25180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } has no permanent_uuid. Determining permanent_uuid...
13:59:50.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.104.60@KRBTEST.COM; defaulting to no policy
13:59:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:59:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.104.60@KRBTEST.COM" created.
13:59:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:50 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283990, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.61@KRBTEST.COM for kudu/127.24.104.62@KRBTEST.COM
13:59:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:59:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.104.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.60.keytab.
13:59:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.104.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.60.keytab.
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.338161 24992 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/logs
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/info.pb
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.60:45263
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.60
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.60.keytab
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.60
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:50.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:50.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:50.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:59:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.344698 25180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } has no permanent_uuid. Determining permanent_uuid...
13:59:50.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:50 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283990, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.61@KRBTEST.COM for kudu/127.24.104.61@KRBTEST.COM
13:59:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.384326 25180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } has no permanent_uuid. Determining permanent_uuid...
13:59:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.387173 25123 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.391567 25180 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.441111 25180 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 1
13:59:50.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.445060 25180 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.513502 25180 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 2
13:59:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.517469 25180 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.608919 25180 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 3
13:59:50.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.612717 25180 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.683323 25188 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.683881 25188 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.684645 25188 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:50.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.717336 25188 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:59:50.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.717578 25188 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:50.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.717841 25188 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:50.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.718051 25188 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:50.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.718240 25188 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:50.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.718425 25188 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:50.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.718562 25188 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38575/jwks.json
13:59:50.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.719684 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } attempt: 7
13:59:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:50 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283988, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.62@KRBTEST.COM for kudu/127.24.104.61@KRBTEST.COM
13:59:50.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.743805 25092 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } has no permanent_uuid. Determining permanent_uuid...
13:59:50.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.749603 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.756168 25180 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 4
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.759794 25180 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.760506 25188 master_runner.cc:386] Master server non-default flags:
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.60.keytab
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.60
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.60:45263
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/info.pb
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.60
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/logs
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.761677 25188 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:50.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.762683 25188 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.782281 25200 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.808002 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 1
13:59:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.812368 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.894029 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 2
13:59:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.899540 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:50.985172 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 3
13:59:50.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.990439 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:51.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:51.059263 25180 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 5
13:59:51.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:51.064610 25180 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:51.130925 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 4
13:59:51.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:51.135826 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:51.419409 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 5
13:59:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:51.423694 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:51.612212 25180 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 6
13:59:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:51.619863 25180 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:51.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:51.976325 25092 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } attempt: 6
13:59:51.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:51.984279 25092 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.104.60:45263: Network error: Client connection negotiation failed: client connection to 127.24.104.60:45263: connect: Connection refused (error 111)
13:59:52.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.783869 25203 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:52.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:50.782682 25201 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:52.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.021257 25202 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:59:52.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.021431 25188 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.025822 25188 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:52.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.028545 25188 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.030015 25188 hybrid_clock.cc:648] HybridClock initialized: now 1786283992029942 us; error 72 us; skew 500 ppm
13:59:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.030869 25188 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283992, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.037637 25188 init.cc:377] Logged in from keytab as kudu/127.24.104.60@KRBTEST.COM (short username kudu)
13:59:52.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.044404 25188 webserver.cc:460] Webserver started at http://127.24.104.60:41495/ using document root <none> and password file <none>
13:59:52.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.045488 25188 fs_manager.cc:359] Metadata directory not provided
13:59:52.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.045725 25188 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:52.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.046195 25188 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:59:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.051512 25188 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/instance:
13:59:52.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c99c1402de445d195579f2888d665e4"
13:59:52.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:52 on dist-test-slave-vxj2"
13:59:52.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.052718 25188 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal/instance:
13:59:52.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c99c1402de445d195579f2888d665e4"
13:59:52.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:52 on dist-test-slave-vxj2"
13:59:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.060482 25188 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
13:59:52.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.066355 25217 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:52.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.067435 25188 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
13:59:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.067806 25188 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c99c1402de445d195579f2888d665e4"
13:59:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:52 on dist-test-slave-vxj2"
13:59:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.068208 25188 fs_report.cc:383] FS layout report
13:59:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/data
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:52.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.116258 25188 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.123381 25188 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.123982 25188 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:52.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.215633 25188 rpc_server.cc:307] RPC server started. Bound to: 127.24.104.60:45263
13:59:52.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.215684 25276 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.104.60:45263 every 8 connection(s)
13:59:52.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.219660 25188 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/info.pb
13:59:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.222872 24992 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 25188
13:59:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.223466 24992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal/instance
13:59:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.227648 25277 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:59:52.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.249343 25277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } has no permanent_uuid. Determining permanent_uuid...
13:59:52.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283992, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.60@KRBTEST.COM for kudu/127.24.104.62@KRBTEST.COM
13:59:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283986, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.104.60@KRBTEST.COM
13:59:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.294245 25277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } has no permanent_uuid. Determining permanent_uuid...
13:59:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283992, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.60@KRBTEST.COM for kudu/127.24.104.61@KRBTEST.COM
13:59:52.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283986, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.104.61@KRBTEST.COM
13:59:52.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.322546 25277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } has no permanent_uuid. Determining permanent_uuid...
13:59:52.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283992, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.60@KRBTEST.COM for kudu/127.24.104.60@KRBTEST.COM
13:59:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283986, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.104.62@KRBTEST.COM
13:59:52.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.352977 25277 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: Bootstrap starting.
13:59:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.353858 24992 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
13:59:52.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.361047 25277 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: Neither blocks nor log segments found. Creating new log.
13:59:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.363740 25277 log.cc:824] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: Log is configured to *not* fsync() on all Append() calls
13:59:52.366 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:52.368 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
13:59:52.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.369835 25277 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: No bootstrap required, opened a new log
13:59:52.382 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
13:59:52.383 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
13:59:52.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.104.62"
port: 33403
}
bound_http_address {
host: "127.24.104.62"
port: 39629
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.104.61"
port: 35389
}
bound_http_address {
host: "127.24.104.61"
port: 37399
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.104.60"
port: 45263
}
bound_http_address {
host: "127.24.104.60"
port: 41495
}
}
}
13:59:52.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:59:52.389 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:52.389 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:59:52.391 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:59:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.401304 25277 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
13:59:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.401927 25277 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:59:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.402124 25277 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c99c1402de445d195579f2888d665e4, State: Initialized, Role: FOLLOWER
13:59:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.402881 25277 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
13:59:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.405706 25288 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } } }
13:59:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.407083 25277 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [sys.catalog]: configured and running, proceeding with master startup.
13:59:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.407156 25288 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [sys.catalog]: This master's current role is: FOLLOWER
13:59:52.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283992, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:52.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:59:52.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:52.438 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc
13:59:52.448 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:59:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.448529 25301 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:52.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.449010 25301 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:52.450 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc
13:59:52.452 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:59:52.457 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
13:59:52.459 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:59:52.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:59:52.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.433902 25188 master_runner.cc:186] Error getting master registration for 127.24.104.62:33403: OK, instance_id { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" instance_seqno: 1786283988331545 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:59:52.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.461042 25188 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:59:52.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:52 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283992, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:52.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:59:52.500 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:52.500 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc
13:59:52.502 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:59:52.503 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.104.60:45263
13:59:52.505 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
13:59:52.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.505893 24992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 25188
13:59:52.531 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:52.531 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.104.61:35389
13:59:52.531 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
13:59:52.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.532037 24992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 25098
13:59:52.562 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:52.562 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.104.62:33403
13:59:52.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
13:59:52.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.563516 24992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 25019
13:59:52.588 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:52.589 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.104.60:45263
13:59:52.590 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:52.592139 24992 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:52.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/logs
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/info.pb
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.60:45263
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.60
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41495
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.60.keytab
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.60
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:59:52.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:59:52.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.932230 25307 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.932896 25307 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.933746 25307 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:52.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.967366 25307 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:59:52.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.967620 25307 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:52.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.967923 25307 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:52.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.968127 25307 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:52.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.968317 25307 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:52.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.968495 25307 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:52.968642 25307 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38575/jwks.json
13:59:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:53.008530 25307 master_runner.cc:386] Master server non-default flags:
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.60.keytab
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.60
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.60:45263
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/info.pb
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.60
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41495
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/logs
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:53.009794 25307 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:53.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:53.010845 25307 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:53.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:53.030371 25314 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:53.030707 25316 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:53.031013 25313 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.340662 25315 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:59:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.340894 25307 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:54.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.345376 25307 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:54.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.348026 25307 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.349404 25307 hybrid_clock.cc:648] HybridClock initialized: now 1786283994349340 us; error 58 us; skew 500 ppm
13:59:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.350344 25307 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:54 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283994, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.356281 25307 init.cc:377] Logged in from keytab as kudu/127.24.104.60@KRBTEST.COM (short username kudu)
13:59:54.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.361688 25307 webserver.cc:460] Webserver started at http://127.24.104.60:41495/ using document root <none> and password file <none>
13:59:54.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.362886 25307 fs_manager.cc:359] Metadata directory not provided
13:59:54.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.363193 25307 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.371429 25307 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
13:59:54.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.376047 25325 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:54.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.377384 25307 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
13:59:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.377790 25307 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c99c1402de445d195579f2888d665e4"
13:59:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:52 on dist-test-slave-vxj2"
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.380584 25307 fs_report.cc:383] FS layout report
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/wal
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/data
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:54.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.464018 25307 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:54.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.469947 25307 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:54.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.470505 25307 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:54.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.546156 25307 rpc_server.cc:307] RPC server started. Bound to: 127.24.104.60:45263
13:59:54.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.546239 25381 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.104.60:45263 every 8 connection(s)
13:59:54.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.549057 25307 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-2/data/info.pb
13:59:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.556759 24992 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 25307
13:59:54.557 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:54.557 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.104.61:35389
13:59:54.557 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.559114 24992 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/logs
13:59:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/info.pb
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.61:35389
13:59:54.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.61
13:59:54.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37399
13:59:54.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.61.keytab
13:59:54.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.61
13:59:54.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:54.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:59:54.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.563289 25382 sys_catalog.cc:263] Verifying existing consensus state
13:59:54.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.568660 25382 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: Bootstrap starting.
13:59:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.603575 25382 log.cc:824] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: Log is configured to *not* fsync() on all Append() calls
13:59:54.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.608165 25382 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:59:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.609015 25382 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: Bootstrap complete.
13:59:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.634403 25382 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
13:59:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.637063 25382 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:59:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.637393 25382 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c99c1402de445d195579f2888d665e4, State: Initialized, Role: FOLLOWER
13:59:54.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.638358 25382 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
13:59:54.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.641403 25387 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } } }
13:59:54.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.642299 25387 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [sys.catalog]: This master's current role is: FOLLOWER
13:59:54.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.643118 25382 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [sys.catalog]: configured and running, proceeding with master startup.
13:59:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.673383 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.673859 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:54.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.676467 25307 master_runner.cc:186] Error getting master registration for 127.24.104.62:33403: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111),
13:59:54.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.707612 25307 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:59:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.725827 25333 tablet.cc:2369] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:59:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.850440 24827 debug-util.cc:398] Leaking SignalData structure 0x7b08000beca0 after lost signal to thread 24696
13:59:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.851583 24827 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5cc0 after lost signal to thread 24830
13:59:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.920655 25384 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.921592 25384 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:54.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.922973 25384 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:54.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.966770 25384 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:59:54.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.967082 25384 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:54.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.967474 25384 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.967809 25384 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:54.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.968138 25384 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:54.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.968441 25384 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:54.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.968670 25384 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38575/jwks.json
13:59:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.977584 25387 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:59:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.978168 25387 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
13:59:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.983716 25329 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111)
13:59:54.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:54.988655 25387 leader_election.cc:288] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403), 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389)
13:59:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:54.998587 25330 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389): Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:55.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:55.000887 25329 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403): Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111)
13:59:55.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:55.001344 25329 leader_election.cc:302] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c99c1402de445d195579f2888d665e4; no voters: 6ba3be881dcf4672990844c9c655cc60, dc708307afc847cfb7289b0d0d8ef187
13:59:55.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:55.002202 25387 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:59:55.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:55.042150 25384 master_runner.cc:386] Master server non-default flags:
13:59:55.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:55.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:55.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:55.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:55.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.61.keytab
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.61
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.61:35389
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:59:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/info.pb
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.61
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37399
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/logs
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:55.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:55.047914 25384 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:55.050274 25384 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:55.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:55.091706 25409 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:55.679116 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:55.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:55.679788 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:56.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:56.684597 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:56.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:56.685364 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:56.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:56.870783 25415 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:59:56.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:56.872159 25415 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
13:59:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:56.886682 25415 leader_election.cc:288] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403), 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389)
13:59:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:56.895906 25329 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403): Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111)
13:59:56.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:56.903260 25330 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389): Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:56.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:56.903914 25330 leader_election.cc:302] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c99c1402de445d195579f2888d665e4; no voters: 6ba3be881dcf4672990844c9c655cc60, dc708307afc847cfb7289b0d0d8ef187
13:59:56.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:56.907317 25415 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:59:57.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:55.098125 25408 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:57.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:55.095605 25411 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:57.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.012272 25410 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1917 milliseconds
13:59:57.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.012444 25384 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:57.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.014195 25384 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.021101 25384 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.022840 25384 hybrid_clock.cc:648] HybridClock initialized: now 1786283997022735 us; error 122 us; skew 500 ppm
13:59:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.024104 25384 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:57 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283997, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.047118 25384 init.cc:377] Logged in from keytab as kudu/127.24.104.61@KRBTEST.COM (short username kudu)
13:59:57.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.081681 25384 webserver.cc:460] Webserver started at http://127.24.104.61:37399/ using document root <none> and password file <none>
13:59:57.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.083465 25384 fs_manager.cc:359] Metadata directory not provided
13:59:57.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.084017 25384 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.103729 25384 fs_manager.cc:709] Time spent opening directory manager: real 0.015s user 0.006s sys 0.009s
13:59:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.111694 25424 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.113381 25384 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.005s sys 0.001s
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.113818 25384 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ba3be881dcf4672990844c9c655cc60"
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:50 on dist-test-slave-vxj2"
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.117635 25384 fs_report.cc:383] FS layout report
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/data
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:57.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.198484 25384 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:57.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.207919 25384 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:57.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.208657 25384 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.366549 25384 rpc_server.cc:307] RPC server started. Bound to: 127.24.104.61:35389
13:59:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.366710 25480 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.104.61:35389 every 8 connection(s)
13:59:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.370842 25384 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/data/info.pb
13:59:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.375432 24992 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 25384
13:59:57.378 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:57.378 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.104.62:33403
13:59:57.378 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.380892 24992 external_mini_cluster.cc:1326] Running /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task77Adji/build/tsan/bin/kudu
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/logs
13:59:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/info.pb
13:59:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:59:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:59:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:59:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:59:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.62:33403
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.62
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39629
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.62.keytab
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.62
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:59:57.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.407806 25481 sys_catalog.cc:263] Verifying existing consensus state
13:59:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 13:59:57.409137 25481 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:59:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.409755 25384 master.cc:537] Master@127.24.104.61:35389 shutting down...
13:59:57.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.539497 25384 master.cc:552] Master@127.24.104.61:35389 shutdown complete.
13:59:57.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:59:57.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.690519 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:57.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.690951 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:57.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.854221 25484 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:59:57.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.854868 25484 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:59:57.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.855849 25484 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:59:57.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.891135 25484 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:59:57.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.891441 25484 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:59:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.891877 25484 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:59:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.892218 25484 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:59:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.892542 25484 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:59:57.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.892918 25484 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:59:57.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.893157 25484 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38575/jwks.json
13:59:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.935921 25484 master_runner.cc:386] Master server non-default flags:
13:59:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:59:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.104.20:36503
13:59:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:59:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:59:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data
13:59:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:59:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:59:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.104.62:33403,127.24.104.61:35389,127.24.104.60:45263
13:59:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/krb5kdc/kudu_127.24.104.62.keytab
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.104.62
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.104.62:33403
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38575/jwks.json
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/info.pb
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.104.62
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39629
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/logs
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:59:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 12:42:27 UTC on ccycloud.quasar-xnesvs.root.comops.site
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575992
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.937469 25484 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:59:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:57.938903 25484 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:59:57.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.958060 25494 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:58.694206 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:58.694558 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:59.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.958451 25493 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:59.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:57.958948 25496 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:59:59.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:59.238221 25495 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:59:59.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.238386 25484 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:59:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.242676 25484 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:59:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.246042 25484 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:59:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.247498 25484 hybrid_clock.cc:648] HybridClock initialized: now 1786283999247430 us; error 79 us; skew 500 ppm
13:59:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.248543 25484 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:59:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:59 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283999, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:59:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.255427 25484 init.cc:377] Logged in from keytab as kudu/127.24.104.62@KRBTEST.COM (short username kudu)
13:59:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.261730 25484 webserver.cc:460] Webserver started at http://127.24.104.62:39629/ using document root <none> and password file <none>
13:59:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.262840 25484 fs_manager.cc:359] Metadata directory not provided
13:59:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.263142 25484 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:59:59.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.272946 25484 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
13:59:59.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.278023 25506 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:59:59.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.279131 25484 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
13:59:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.279552 25484 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data,/tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc708307afc847cfb7289b0d0d8ef187"
13:59:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 13:59:48 on dist-test-slave-vxj2"
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.282546 25484 fs_report.cc:383] FS layout report
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/data
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:59:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:59:59.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.326540 25484 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:59:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.332791 25484 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:59:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.333531 25484 kserver.cc:163] Server-wide thread pool size limit: 3276
13:59:59.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.414978 25484 rpc_server.cc:307] RPC server started. Bound to: 127.24.104.62:33403
13:59:59.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.415117 25560 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.104.62:33403 every 8 connection(s)
13:59:59.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.417928 25484 server_base.cc:1166] Dumped server information to /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/data/info.pb
13:59:59.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:59:59.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.420496 24992 external_mini_cluster.cc:1388] Started /tmp/dist-test-task77Adji/build/tsan/bin/kudu as pid 25484
13:59:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.430615 25561 sys_catalog.cc:263] Verifying existing consensus state
13:59:59.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 13:59:59.431622 25561 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:59:59.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.432160 25484 master.cc:537] Master@127.24.104.62:33403 shutting down...
13:59:59.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.468242 25564 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:59:59.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.468767 25564 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
13:59:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.478761 25564 leader_election.cc:288] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403), 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389)
13:59:59.501 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.104.60:45263(127.24.104.60:45263)] connecting to peer
13:59:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:59.506417 25330 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389): Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
13:59:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:59.507490 25329 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403): Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111)
13:59:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.507905 25329 leader_election.cc:302] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c99c1402de445d195579f2888d665e4; no voters: 6ba3be881dcf4672990844c9c655cc60, dc708307afc847cfb7289b0d0d8ef187
13:59:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.508725 25564 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:59:59.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.517094 25484 master.cc:552] Master@127.24.104.62:33403 shutdown complete.
13:59:59.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task77Adji/test-tmp/mini-kudu-cluster5222806515812935728/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:59:59.570 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
13:59:59.571 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
13:59:59.636 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.104.60:45263(127.24.104.60:45263)] Successfully connected to peer
13:59:59.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
13:59:59.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
13:59:59.643 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
13:59:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:59.697470 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:59:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 13:59:59.697718 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:59:59.703 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
13:59:59.703 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
13:59:59.704 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
13:59:59.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.710183 25307 master_runner.cc:186] Error getting master registration for 127.24.104.62:33403: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111),
13:59:59.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 13:59:59.710650 25307 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:59:59.722 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
13:59:59.732 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.24.104.60
13:59:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 13:59:59 dist-test-slave-vxj2 krb5kdc[25005](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786283992, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.104.60@KRBTEST.COM
13:59:59.984 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3dbb7318, L:/127.0.0.1:44196 - R:/127.24.104.60:45263] using SASL/GSSAPI
14:00:00.003 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.010 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.012 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.017 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.017 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.017 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.018 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.028 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.029 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.033 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.033 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.034 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.035 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.048 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.050 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.050 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.054 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.055 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.055 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.056 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.068 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.069 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.070 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.070 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.071 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.073 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.074 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.074 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.074 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.088 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.089 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.093 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.094 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.096 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.098 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.169 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.174 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.174 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.174 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.175 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.268 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.270 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.270 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.274 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.274 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.275 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.276 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.548 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.554 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.554 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.554 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.555 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.559 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:00.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:00.700649 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:00.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:00.700929 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:00.908 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:00.910 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:00.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:00.914 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.914 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.914 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:00.915 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:01.148 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:01.150 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:01.150 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:01.151 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:01.152 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:01.154 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:01.154 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:01.154 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:01.154 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:01.488 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:01.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:01.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:01.491 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:01.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:01.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:01.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:01.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:01.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:01.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:01.704010 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:01.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:01.704275 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:02.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:02.489 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:02.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:02.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:02.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:02.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:02.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:02.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:02.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:02.707274 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:02.707572 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:03.088 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:03.089 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:03.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:03.090 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:03.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:03.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:03.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:03.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:03.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:03.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:03.598065 25579 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:00:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:03.598630 25579 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
14:00:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:03.600522 25579 leader_election.cc:288] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403), 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389)
14:00:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:03.603204 25329 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111) [suppressed 5 similar messages]
14:00:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:03.606356 25330 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389): Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
14:00:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:03.606873 25329 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403): Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111)
14:00:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:03.607304 25329 leader_election.cc:302] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c99c1402de445d195579f2888d665e4; no voters: 6ba3be881dcf4672990844c9c655cc60, dc708307afc847cfb7289b0d0d8ef187
14:00:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:03.607995 25579 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:00:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:03.710920 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:03.711269 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:04.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 14:00:04 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786284004, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:04.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:04.360303 25321 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:04.628 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:04.629 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:04.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:04.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:04.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:04.631 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:04.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:04.631 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:04.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:04.714195 25307 master_runner.cc:186] Error getting master registration for 127.24.104.62:33403: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111),
14:00:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:04.714681 25307 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:00:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:04.715430 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:04.715694 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:04.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:04.856730 24310 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac040 after lost signal to thread 24242
14:00:04.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:04.857353 24310 debug-util.cc:398] Leaking SignalData structure 0x7b08000991c0 after lost signal to thread 24313
14:00:05.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:05.464094 24974 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba300 after lost signal to thread 24843
14:00:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:05.465433 24974 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba020 after lost signal to thread 24977
14:00:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:05.718688 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:05.719002 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:06.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:06.722445 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:06.722813 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:07.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:07.725703 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:07.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:07.726003 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:07.808 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:07.810 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:07.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:07.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:07.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:07.813 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:07.814 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:07.814 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:07.814 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:08.728969 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:08.729295 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:09.717531 25307 master_runner.cc:186] Error getting master registration for 127.24.104.62:33403: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111),
14:00:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:09.717922 25307 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:00:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:09.732450 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:09.732769 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:09.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:09.884279 25588 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:00:09.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:09.884663 25588 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
14:00:09.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:09.886344 25588 leader_election.cc:288] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403), 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389)
14:00:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:09.887882 25329 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:00:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:09.890998 25329 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403): Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111)
14:00:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:09.891417 25330 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389): Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
14:00:09.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:09.891712 25330 leader_election.cc:302] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c99c1402de445d195579f2888d665e4; no voters: 6ba3be881dcf4672990844c9c655cc60, dc708307afc847cfb7289b0d0d8ef187
14:00:09.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:09.892261 25588 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:00:10.008 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:10.010 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:10.010 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:10.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:10.011 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:10.013 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:10.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:10.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:10.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:10.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:10.735970 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:10.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:10.736232 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:11.739120 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:11.739391 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:12.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:12.742202 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:12.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:12.742467 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:12.988 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:12.989 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:12.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:12.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:12.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:12.995 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:12.995 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:12.995 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:12.995 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:13.337055 25599 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:00:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:13.337775 25599 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
14:00:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:13.340164 25599 leader_election.cc:288] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403), 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389)
14:00:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:13.347544 25329 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403): Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111)
14:00:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:13.348281 25330 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389): Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
14:00:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:13.348824 25330 leader_election.cc:302] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c99c1402de445d195579f2888d665e4; no voters: 6ba3be881dcf4672990844c9c655cc60, dc708307afc847cfb7289b0d0d8ef187
14:00:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:13.350055 25599 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:00:13.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:13.746359 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:13.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:13.746692 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:14.288 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:14.290 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:14.290 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:14.291 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:14.291 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:14.293 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:14.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:14.294 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:14.294 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:14.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 14:00:14 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786284014, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:14.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:14.362993 25321 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:14.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:14.721045 25307 master_runner.cc:186] Error getting master registration for 127.24.104.62:33403: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111),
14:00:14.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:14.721585 25307 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:00:14.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:14.749886 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:14.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:14.750240 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:15.753840 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:15.754158 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:16.288 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:16.290 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:16.290 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:16.290 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:16.291 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:16.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:16.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:16.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:16.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:16.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:16.757133 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:16.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:16.757510 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:17.760447 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:17.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:17.760757 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:17.807 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:17.808 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:17.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:17.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:17.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:17.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:17.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:17.810 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:17.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:18.763780 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:18.764119 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:19.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:19.724200 25307 master_runner.cc:186] Error getting master registration for 127.24.104.62:33403: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111),
14:00:19.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:19.724606 25307 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:00:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:19.767138 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:19.767458 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:20.770407 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:20.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:20.770723 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:20.928 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:20.929 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:20.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:20.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:20.930 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:20.930 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:20.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:20.931 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:20.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:21.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:21.773674 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:21.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:21.773972 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:22.428 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:22.428 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:22.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:22.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:22.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:22.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:22.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:22.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:22.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:22.777000 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:22.777416 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:23.128 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:23.129 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:23.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:23.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:23.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:23.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:23.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:23.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:23.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:23.718262 24389 debug-util.cc:398] Leaking SignalData structure 0x7b080009c520 after lost signal to thread 24320
14:00:23.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:23.719108 24389 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1400 after lost signal to thread 24392
14:00:23.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:23.780938 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:23.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:23.781427 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:23.948 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:23.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:23.950 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:23.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:23.951 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:23.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:23.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:23.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:23.953 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:24.234747 25618 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:00:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:24.235383 25618 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc708307afc847cfb7289b0d0d8ef187" member_type: VOTER last_known_addr { host: "127.24.104.62" port: 33403 } } peers { permanent_uuid: "6ba3be881dcf4672990844c9c655cc60" member_type: VOTER last_known_addr { host: "127.24.104.61" port: 35389 } } peers { permanent_uuid: "4c99c1402de445d195579f2888d665e4" member_type: VOTER last_known_addr { host: "127.24.104.60" port: 45263 } }
14:00:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:24.237617 25618 leader_election.cc:288] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403), 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389)
14:00:24.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:24.240396 25329 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111) [suppressed 3 similar messages]
14:00:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:24.243981 25330 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6ba3be881dcf4672990844c9c655cc60 (127.24.104.61:35389): Network error: Client connection negotiation failed: client connection to 127.24.104.61:35389: connect: Connection refused (error 111)
14:00:24.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:24.244611 25329 leader_election.cc:334] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer dc708307afc847cfb7289b0d0d8ef187 (127.24.104.62:33403): Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111)
14:00:24.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:24.245064 25329 leader_election.cc:302] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c99c1402de445d195579f2888d665e4; no voters: 6ba3be881dcf4672990844c9c655cc60, dc708307afc847cfb7289b0d0d8ef187
14:00:24.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:24.245957 25618 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:00:24.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 14:00:24 dist-test-slave-vxj2 krb5kdc[25005](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786284024, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.104.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:00:24.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:24.365567 25321 init.cc:295] Successfully reacquired a new kerberos TGT
14:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:24.727181 25307 master_runner.cc:186] Error getting master registration for 127.24.104.62:33403: Network error: Client connection negotiation failed: client connection to 127.24.104.62:33403: connect: Connection refused (error 111),
14:00:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:24.727607 25307 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:00:24.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:24.784446 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:24.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:24.784705 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:25.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:25.489 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:25.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:25.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:25.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:25.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:25.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:25.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:25.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:25.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:25.787364 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:25.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:25.787631 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:26.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:26.790414 25398 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4c99c1402de445d195579f2888d665e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:00:26.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:26.790676 25398 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:00:27.267 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.61:35389(127.24.104.61:35389)] connecting to peer
14:00:27.268 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.104.62:33403(127.24.104.62:33403)] connecting to peer
14:00:27.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:27.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.104.61:35389(127.24.104.61:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:27.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.61:35389: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389
14:00:27.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:27.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.62:33403(127.24.104.62:33403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:27.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.104.62:33403: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
14:00:27.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403; will retry
14:00:27.276 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27846), Traces: [0ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [207ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [270ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [565ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [584ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [593ms] refreshing cache from master, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [597ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [598ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [599ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [601ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [612ms] refreshing cache from master, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [616ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [620ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [622ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [633ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [639ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [640ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [641ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [653ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [661ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [733ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [738ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [740ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [741ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [833ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [837ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [839ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [840ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [842ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1113ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1126ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1473ms] refreshing cache from master, [1473ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1474ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1477ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1478ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1481ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1713ms] refreshing cache from master, [1713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1716ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1718ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1720ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1720ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2053ms] refreshing cache from master, [2053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [2054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [2059ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2059ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [3052ms] refreshing cache from master, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [3054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [3056ms] too many traces: truncated at 100 traces, deferred=Deferred@427722069(state=PENDING, result=null, callback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882), errback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882)))
14:00:27.283 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.104.62:33403(127.24.104.62:33403)] disconnecting while in state TERMINATED
14:00:27.284 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.104.60:45263(127.24.104.60:45263)] disconnecting while in state READY
14:00:27.284 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.104.61:35389(127.24.104.61:35389)] disconnecting while in state TERMINATED
14:00:27.289 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:00:27.291 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.104.60:45263(127.24.104.60:45263)] handling channelInactive
14:00:27.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.104.60:45263(127.24.104.60:45263)] cleaning up while in state READY due to: connection closed
14:00:27.293 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:00:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:27.295879 24992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 25307
14:00:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T14:00:28Z chronyd exiting
14:00:29.751 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:00:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:29.753448 24228 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 24525
14:00:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:29.795387 24228 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 24690
14:00:29.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:29.827519 24228 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 24842
14:00:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:29.860976 24228 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 24241
14:00:29.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:29.899497 24228 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 24316
14:00:29.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:29.931856 24424 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.169.62:37917: connect: Connection refused (error 111)
14:00:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 14:00:29.936059 24424 consensus_peers.cc:487] T 00000000000000000000000000000000 P 25b61aa1960c454da7cc2cf40ca44277 -> Peer 408ab6b682b0482b93619159e4d1c039 (127.23.169.62:37917): Couldn't send request to peer 408ab6b682b0482b93619159e4d1c039. Status: Network error: Client connection negotiation failed: client connection to 127.23.169.62:37917: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:00:29.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 14:00:29.948766 24228 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task77Adji/build/tsan/bin/kudu with pid 24395
14:00:31.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T14:00:31Z chronyd exiting
14:00:32.216 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27846), Traces: [0ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [207ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [270ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [565ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [584ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [593ms] refreshing cache from master, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [597ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [598ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [599ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [601ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [612ms] refreshing cache from master, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [616ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [620ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [622ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [633ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [639ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [640ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [641ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [653ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [661ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [733ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [738ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [740ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [741ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [833ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [837ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [839ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [840ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [842ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1113ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1126ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1473ms] refreshing cache from master, [1473ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1474ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1477ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1478ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1481ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1713ms] refreshing cache from master, [1713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1716ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1718ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1720ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1720ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2053ms] refreshing cache from master, [2053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [2054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [2059ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2059ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [3052ms] refreshing cache from master, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [3054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [3056ms] too many traces: truncated at 100 traces, deferred=Deferred@427722069(state=PENDING, result=null, callback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882), errback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.104.62:33403
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27846), Traces: [0ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [207ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [270ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [565ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [584ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [593ms] refreshing cache from master, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [597ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [598ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [599ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [601ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [612ms] refreshing cache from master, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [616ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [620ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [622ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [633ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [639ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [640ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [641ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [653ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [661ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [733ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [738ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [740ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [741ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [833ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [837ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [839ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [840ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [842ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1113ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1126ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1473ms] refreshing cache from master, [1473ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1474ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1477ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1478ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1481ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1713ms] refreshing cache from master, [1713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1716ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1718ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1720ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1720ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2053ms] refreshing cache from master, [2053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [2054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [2059ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2059ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [3052ms] refreshing cache from master, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [3054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [3056ms] too many traces: truncated at 100 traces, deferred=Deferred@427722069(state=PENDING, result=null, callback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882), errback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.104.62:33403
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
14:00:32.677 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
14:00:32.677 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27846), Traces: [0ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [207ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [270ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [565ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [584ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [593ms] refreshing cache from master, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [597ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [598ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [599ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [601ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [612ms] refreshing cache from master, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [616ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [620ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [622ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [633ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [639ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [640ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [641ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [653ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [661ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [733ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [738ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [740ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [741ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [833ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [837ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [839ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [840ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [842ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1113ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1126ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1473ms] refreshing cache from master, [1473ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1474ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1477ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1478ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1481ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1713ms] refreshing cache from master, [1713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1716ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1718ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1720ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1720ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2053ms] refreshing cache from master, [2053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [2054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [2059ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2059ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [3052ms] refreshing cache from master, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [3054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [3056ms] too many traces: truncated at 100 traces, deferred=Deferred@427722069(state=PENDING, result=null, callback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882), errback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.104.62:33403
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27846), Traces: [0ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [207ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [270ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [565ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [582ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [584ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [593ms] refreshing cache from master, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [594ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [597ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [598ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [599ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [601ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [612ms] refreshing cache from master, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [616ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [620ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [622ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [633ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [639ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [640ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [641ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [653ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [661ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [733ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [738ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [740ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [741ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [833ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [837ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [839ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [840ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [842ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1113ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1126ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1473ms] refreshing cache from master, [1473ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1474ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1477ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1478ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1481ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1713ms] refreshing cache from master, [1713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [1714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [1716ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [1718ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [1720ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [1720ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2053ms] refreshing cache from master, [2053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [2054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.61:35389: Network error: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [2057ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.60:45263: OK, [2059ms] Sub RPC ConnectToMaster: received response from server master-127.24.104.62:33403: Network error: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [2059ms] delaying RPC due to: Service unavailable: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403, [3052ms] refreshing cache from master, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.60:45263, [3053ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.61:35389, [3054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.104.62:33403, [3056ms] too many traces: truncated at 100 traces, deferred=Deferred@427722069(state=PENDING, result=null, callback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882), errback=(continuation of Deferred@1916127147 after retry RPC after error@1248901359) -> (continuation of Deferred@268169957 after retry RPC after error@1648532066) -> (continuation of Deferred@1041629470 after retry RPC after error@1903063539) -> (continuation of Deferred@559077857 after retry RPC after error@1471534627) -> (continuation of Deferred@1495606074 after retry RPC after error@1531322372) -> (continuation of Deferred@1827436380 after retry RPC after error@361865260) -> (continuation of Deferred@1709014359 after retry RPC after error@259074024) -> (continuation of Deferred@11069389 after retry RPC after error@404793140) -> (continuation of Deferred@370755234 after retry RPC after error@282164718) -> (continuation of Deferred@1468032694 after retry RPC after error@1241801333) -> (continuation of Deferred@376705473 after retry RPC after error@231809260) -> (continuation of Deferred@1504727606 after retry RPC after error@818840767) -> (continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.104.60:45263,127.24.104.61:35389,127.24.104.62:33403) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.61:35389(127.24.104.61:35389): Connection refused: /127.24.104.61:35389,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.104.62:33403(127.24.104.62:33403): Connection refused: /127.24.104.62:33403
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.104.62:33403
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)