18:47:52.279 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
18:47:52.291 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskvcoJf5/test-tmp
18:47:52.293 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
18:47:52.297 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskvcoJf5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
18:47:52.582 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
18:47:52.716 [DEBUG - main] (MiniKuduCluster.java:195) Response:
18:47:52.717 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
18:47:52.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:52.720359 20575 mini_oidc.cc:152] Starting JWKS server
18:47:52.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:52.724131 20575 webserver.cc:460] Webserver started at http://127.0.0.1:45743/ using document root <none> and password file <none>
18:47:52.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:52.724625 20575 mini_oidc.cc:189] Starting OIDC Discovery server
18:47:52.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:52.726732 20575 webserver.cc:460] Webserver started at http://127.0.0.1:34847/ using document root <none> and password file <none>
18:47:52.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:47:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
18:47:52.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:47:52Z Disabled control of system clock
18:47:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:52.777549 20575 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:47:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:47:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/wal
18:47:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/data
18:47:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:47:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:47:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:47:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:47:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:47:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:47:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/logs
18:47:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/data/info.pb
18:47:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:47:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:47:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:47:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:47:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:47:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:47:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:47:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:47:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:47:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:47:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:47:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:47:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.254:36031
18:47:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.254
18:47:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:47:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:47:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:47:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:47:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:47:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:47:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:47:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:47:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45743/jwks.json with env {}
18:47:53.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.121209 20587 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:47:53.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.121906 20587 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:47:53.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.122871 20587 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:47:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.163444 20587 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:47:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.163787 20587 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:47:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.163982 20587 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:47:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.164192 20587 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:47:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.164462 20587 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:47:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.164680 20587 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45743/jwks.json
18:47:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:53.204860 20587 master_runner.cc:386] Master server non-default flags:
18:47:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:47:53.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:47:53.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:47:53.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:47:53.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/data
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/wal
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.254:36031
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:47:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45743/jwks.json
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/data/info.pb
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.254
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/logs
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:47:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:53.206629 20587 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:47:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:53.209189 20587 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:47:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.232985 20594 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.233047 20593 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:53.233038 20596 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:53.234225 20587 server_base.cc:1034] running on GCE node
18:47:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.335208 20587 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:47:54.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.338269 20587 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:47:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.339694 20587 hybrid_clock.cc:648] HybridClock initialized: now 1781894874339626 us; error 83 us; skew 500 ppm
18:47:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.340529 20587 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:47:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.349727 20587 webserver.cc:460] Webserver started at http://127.20.23.254:42731/ using document root <none> and password file <none>
18:47:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.350687 20587 fs_manager.cc:359] Metadata directory not provided
18:47:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.350876 20587 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:47:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.351256 20587 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:47:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.356036 20587 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/data/instance:
18:47:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2b1eb1b95344d6b93c0721a08891cce"
18:47:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:54 on dist-test-slave-9qt6"
18:47:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.357105 20587 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/wal/instance:
18:47:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2b1eb1b95344d6b93c0721a08891cce"
18:47:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:54 on dist-test-slave-9qt6"
18:47:54.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.364603 20587 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
18:47:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.370410 20603 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:47:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.371850 20587 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
18:47:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.372159 20587 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/wal
18:47:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2b1eb1b95344d6b93c0721a08891cce"
18:47:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:54 on dist-test-slave-9qt6"
18:47:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.372469 20587 fs_report.cc:383] FS layout report
18:47:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:47:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/wal
18:47:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/wal
18:47:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/data/data
18:47:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:47:54.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:47:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.414027 20587 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:47:54.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.415467 20587 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:47:54.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.415846 20587 kserver.cc:163] Server-wide thread pool size limit: 3276
18:47:54.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.487033 20587 rpc_server.cc:307] RPC server started. Bound to: 127.20.23.254:36031
18:47:54.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.487208 20654 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.23.254:36031 every 8 connection(s)
18:47:54.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.489645 20587 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/data/info.pb
18:47:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.492619 20575 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 20587
18:47:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.493155 20575 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-0/wal/instance
18:47:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.496156 20655 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:47:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.496565 20575 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:47:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/wal
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/data
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/logs
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/data/info.pb
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:47:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:47:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.253:44071
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.253
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:47:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:47:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45743/jwks.json with env {}
18:47:54.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.514083 20655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } has no permanent_uuid. Determining permanent_uuid...
18:47:54.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.547114 20655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
18:47:54.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.550308 20605 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.23.253:44071: connect: Connection refused (error 111)
18:47:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.554752 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.23.253:44071: connect: Connection refused (error 111)
18:47:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.604265 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } attempt: 1
18:47:54.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.608457 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.23.253:44071: connect: Connection refused (error 111)
18:47:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.676857 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } attempt: 2
18:47:54.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.681046 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.23.253:44071: connect: Connection refused (error 111)
18:47:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.772460 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } attempt: 3
18:47:54.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.777109 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.23.253:44071: connect: Connection refused (error 111)
18:47:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.837338 20657 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:47:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.837903 20657 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:47:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.838442 20657 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:47:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.868929 20657 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:47:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.869207 20657 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:47:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.869402 20657 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:47:54.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.869591 20657 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:47:54.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.869765 20657 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:47:54.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.869911 20657 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45743/jwks.json
18:47:54.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.905073 20657 master_runner.cc:386] Master server non-default flags:
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/data
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/wal
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:47:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.253:44071
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45743/jwks.json
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/data/info.pb
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.253
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/logs
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:47:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:47:54.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:47:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:47:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:47:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:47:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:47:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:47:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.906409 20657 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:47:54.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.908514 20657 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:47:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.920544 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } attempt: 4
18:47:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.925940 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.23.253:44071: connect: Connection refused (error 111)
18:47:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.928740 20669 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:55.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:55.225544 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } attempt: 5
18:47:55.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:55.229962 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.23.253:44071: connect: Connection refused (error 111)
18:47:55.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:55.777597 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } attempt: 6
18:47:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:55.785063 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.23.253:44071: connect: Connection refused (error 111)
18:47:56.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.928797 20666 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:56.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:54.928818 20667 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:56.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:54.930305 20657 server_base.cc:1034] running on GCE node
18:47:56.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.037166 20657 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:47:56.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.041363 20657 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:47:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.043032 20657 hybrid_clock.cc:648] HybridClock initialized: now 1781894876042968 us; error 90 us; skew 500 ppm
18:47:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.044319 20657 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:47:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.055591 20657 webserver.cc:460] Webserver started at http://127.20.23.253:44055/ using document root <none> and password file <none>
18:47:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.056586 20657 fs_manager.cc:359] Metadata directory not provided
18:47:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.056785 20657 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:47:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.057188 20657 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:47:56.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.062039 20657 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/data/instance:
18:47:56.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "892403f59bc14d3e9e75700ae62186ab"
18:47:56.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:56 on dist-test-slave-9qt6"
18:47:56.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.063238 20657 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/wal/instance:
18:47:56.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "892403f59bc14d3e9e75700ae62186ab"
18:47:56.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:56 on dist-test-slave-9qt6"
18:47:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.070724 20657 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
18:47:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.076939 20678 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:47:56.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.078331 20657 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
18:47:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.078740 20657 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/wal
18:47:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "892403f59bc14d3e9e75700ae62186ab"
18:47:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:56 on dist-test-slave-9qt6"
18:47:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.079111 20657 fs_report.cc:383] FS layout report
18:47:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:47:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/wal
18:47:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/wal
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/data/data
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:47:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:47:56.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:47:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.127312 20657 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:47:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.128896 20657 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:47:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.129431 20657 kserver.cc:163] Server-wide thread pool size limit: 3276
18:47:56.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.210800 20657 rpc_server.cc:307] RPC server started. Bound to: 127.20.23.253:44071
18:47:56.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.210985 20729 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.23.253:44071 every 8 connection(s)
18:47:56.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.213694 20657 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/data/info.pb
18:47:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.219448 20575 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 20657
18:47:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.219779 20730 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:47:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.219856 20575 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-1/wal/instance
18:47:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.222172 20575 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:47:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:47:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/wal
18:47:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/data
18:47:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:47:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:47:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/logs
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/data/info.pb
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.252:40073
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.252
18:47:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:47:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:47:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:47:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:47:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:47:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:47:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:47:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:47:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45743/jwks.json with env {}
18:47:56.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.236688 20730 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } has no permanent_uuid. Determining permanent_uuid...
18:47:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.265796 20730 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
18:47:56.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.283147 20730 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } has no permanent_uuid. Determining permanent_uuid...
18:47:56.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.285689 20680 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:56.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.294346 20730 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:56.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.343935 20730 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 1
18:47:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.348690 20730 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:56.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.417132 20730 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 2
18:47:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.421926 20730 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:56.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.513329 20730 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 3
18:47:56.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.517524 20730 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:56.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.614846 20732 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:47:56.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.615537 20732 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:47:56.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.616025 20732 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:47:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.654388 20732 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:47:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.654743 20732 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:47:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.655030 20732 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:47:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.655246 20732 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:47:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.655453 20732 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:47:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.655588 20732 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45743/jwks.json
18:47:56.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.660987 20730 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 4
18:47:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.666401 20730 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.701021 20732 master_runner.cc:386] Master server non-default flags:
18:47:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:47:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:47:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:47:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:47:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/data
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/wal
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.252:40073
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45743/jwks.json
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/data/info.pb
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.252
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:47:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/logs
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:47:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:47:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:47:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:47:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:47:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.702692 20732 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:47:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.705549 20732 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:47:56.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.727564 20742 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.845794 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } attempt: 7
18:47:56.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.862686 20655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } has no permanent_uuid. Determining permanent_uuid...
18:47:56.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.869329 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.927892 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 1
18:47:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.932955 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:56.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.965883 20730 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 5
18:47:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.970150 20730 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:57.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.014540 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 2
18:47:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:57.018931 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.104557 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 3
18:47:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:57.109127 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:57.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.249841 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 4
18:47:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:57.255182 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:57.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.517792 20730 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 6
18:47:57.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:57.526315 20730 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:57.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.538702 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 5
18:47:57.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:57.544086 20655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.23.252:40073: Network error: Client connection negotiation failed: client connection to 127.20.23.252:40073: connect: Connection refused (error 111)
18:47:57.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.727934 20743 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:57.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:56.731069 20732 server_base.cc:1034] running on GCE node
18:47:57.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:56.730346 20745 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:47:57.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.860201 20732 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:47:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.864262 20732 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:47:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.865875 20732 hybrid_clock.cc:648] HybridClock initialized: now 1781894877865785 us; error 79 us; skew 500 ppm
18:47:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.867084 20732 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:47:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.875425 20732 webserver.cc:460] Webserver started at http://127.20.23.252:38771/ using document root <none> and password file <none>
18:47:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.876542 20732 fs_manager.cc:359] Metadata directory not provided
18:47:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.876770 20732 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:47:57.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.877211 20732 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:47:57.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.885203 20732 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/data/instance:
18:47:57.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0153a5e56be49749573e1d0648c15de"
18:47:57.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:57 on dist-test-slave-9qt6"
18:47:57.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.887207 20732 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/wal/instance:
18:47:57.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0153a5e56be49749573e1d0648c15de"
18:47:57.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:57 on dist-test-slave-9qt6"
18:47:57.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.901163 20732 fs_manager.cc:691] Time spent creating directory manager: real 0.013s user 0.001s sys 0.012s
18:47:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.908892 20757 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:47:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.910334 20732 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
18:47:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.911276 20732 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/wal
18:47:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0153a5e56be49749573e1d0648c15de"
18:47:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:47:57 on dist-test-slave-9qt6"
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.911929 20732 fs_report.cc:383] FS layout report
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/wal
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/wal
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/data/data
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:47:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:47:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:47:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:47:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:47:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:47:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:47:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:47:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:47:57.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.955171 20732 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:47:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.957111 20732 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:47:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:57.957703 20732 kserver.cc:163] Server-wide thread pool size limit: 3276
18:47:58.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.047261 20732 rpc_server.cc:307] RPC server started. Bound to: 127.20.23.252:40073
18:47:58.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.047348 20808 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.23.252:40073 every 8 connection(s)
18:47:58.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.051553 20732 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/data/info.pb
18:47:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.058750 20575 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 20732
18:47:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.058940 20809 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:47:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.059168 20575 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/master-2/wal/instance
18:47:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.077996 20809 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } has no permanent_uuid. Determining permanent_uuid...
18:47:58.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.096558 20655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 6
18:47:58.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.110918 20809 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
18:47:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.128089 20809 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } has no permanent_uuid. Determining permanent_uuid...
18:47:58.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.131453 20655 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce: Bootstrap starting.
18:47:58.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.138805 20655 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce: Neither blocks nor log segments found. Creating new log.
18:47:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.141455 20655 log.cc:824] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce: Log is configured to *not* fsync() on all Append() calls
18:47:58.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.146853 20655 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce: No bootstrap required, opened a new log
18:47:58.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.153229 20809 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de: Bootstrap starting.
18:47:58.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.163863 20809 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de: Neither blocks nor log segments found. Creating new log.
18:47:58.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.169245 20809 log.cc:824] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de: Log is configured to *not* fsync() on all Append() calls
18:47:58.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.178958 20809 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de: No bootstrap required, opened a new log
18:47:58.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.179921 20655 raft_consensus.cc:348] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.181506 20655 raft_consensus.cc:374] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:47:58.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.181877 20655 raft_consensus.cc:729] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2b1eb1b95344d6b93c0721a08891cce, State: Initialized, Role: FOLLOWER
18:47:58.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.183101 20655 consensus_queue.cc:260] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.187215 20821 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.188385 20821 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: This master's current role is: FOLLOWER
18:47:58.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.189538 20655 sys_catalog.cc:564] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: configured and running, proceeding with master startup.
18:47:58.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.212776 20809 raft_consensus.cc:348] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.214335 20809 raft_consensus.cc:374] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:47:58.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.215894 20809 raft_consensus.cc:729] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d0153a5e56be49749573e1d0648c15de, State: Initialized, Role: FOLLOWER
18:47:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.217588 20809 consensus_queue.cc:260] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.222213 20833 sys_catalog.cc:455] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.223668 20833 sys_catalog.cc:458] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [sys.catalog]: This master's current role is: FOLLOWER
18:47:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:58.229543 20832 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:47:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:58.230298 20832 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:47:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.224511 20809 sys_catalog.cc:564] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [sys.catalog]: configured and running, proceeding with master startup.
18:47:58.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.244568 20821 raft_consensus.cc:482] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:47:58.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.245108 20821 raft_consensus.cc:504] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.247776 20821 leader_election.cc:288] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 892403f59bc14d3e9e75700ae62186ab (127.20.23.253:44071), d0153a5e56be49749573e1d0648c15de (127.20.23.252:40073)
18:47:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.254009 20784 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2b1eb1b95344d6b93c0721a08891cce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d0153a5e56be49749573e1d0648c15de" is_pre_election: true
18:47:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.255136 20784 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2b1eb1b95344d6b93c0721a08891cce in term 0.
18:47:58.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.258551 20605 leader_election.cc:302] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a2b1eb1b95344d6b93c0721a08891cce, d0153a5e56be49749573e1d0648c15de; no voters:
18:47:58.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.259547 20821 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 0 FOLLOWER]: Leader pre-election won for term 1
18:47:58.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.259907 20821 raft_consensus.cc:482] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:47:58.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.260221 20821 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 0 FOLLOWER]: Advancing to term 1
18:47:58.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.228127 20587 master_runner.cc:186] Error getting master registration for 127.20.23.253:44071: OK, instance_id { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" instance_seqno: 1781894876189678 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:47:58.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.262712 20587 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:47:58.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.267463 20821 raft_consensus.cc:504] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.271243 20784 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2b1eb1b95344d6b93c0721a08891cce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d0153a5e56be49749573e1d0648c15de"
18:47:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.271900 20784 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [term 0 FOLLOWER]: Advancing to term 1
18:47:58.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.279726 20784 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2b1eb1b95344d6b93c0721a08891cce in term 1.
18:47:58.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.269668 20821 leader_election.cc:288] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [CANDIDATE]: Term 1 election: Requested vote from peers 892403f59bc14d3e9e75700ae62186ab (127.20.23.253:44071), d0153a5e56be49749573e1d0648c15de (127.20.23.252:40073)
18:47:58.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.285493 20605 leader_election.cc:302] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a2b1eb1b95344d6b93c0721a08891cce, d0153a5e56be49749573e1d0648c15de; no voters:
18:47:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.287401 20821 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 1 FOLLOWER]: Leader election won for term 1
18:47:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.290889 20821 raft_consensus.cc:686] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [term 1 LEADER]: Becoming Leader. State: Replica: a2b1eb1b95344d6b93c0721a08891cce, State: Running, Role: LEADER
18:47:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.292479 20821 consensus_queue.cc:237] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.318943 20839 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: SysCatalogTable state changed. Reason: New leader a2b1eb1b95344d6b93c0721a08891cce. Latest consensus state: current_term: 1 leader_uuid: "a2b1eb1b95344d6b93c0721a08891cce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.320492 20839 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: This master's current role is: LEADER
18:47:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.329456 20847 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
18:47:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:58.351357 20848 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:47:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:58.351729 20848 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:47:58.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.358652 20847 catalog_manager.cc:1484] Initializing Kudu cluster ID...
18:47:58.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.400516 20784 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [term 1 FOLLOWER]: Refusing update from remote peer a2b1eb1b95344d6b93c0721a08891cce: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:47:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.402693 20839 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [LEADER]: Connected to new peer: Peer: permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:47:58.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.367513 20732 master_runner.cc:186] Error getting master registration for 127.20.23.253:44071: OK, instance_id { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" instance_seqno: 1781894876189678 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:47:58.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.405977 20732 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:47:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:58.426054 20609 tablet.cc:2369] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:47:58.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.444559 20833 sys_catalog.cc:455] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [sys.catalog]: SysCatalogTable state changed. Reason: New leader a2b1eb1b95344d6b93c0721a08891cce. Latest consensus state: current_term: 1 leader_uuid: "a2b1eb1b95344d6b93c0721a08891cce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.445459 20833 sys_catalog.cc:458] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [sys.catalog]: This master's current role is: FOLLOWER
18:47:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.459543 20839 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a2b1eb1b95344d6b93c0721a08891cce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.460305 20839 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: This master's current role is: LEADER
18:47:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.463876 20847 catalog_manager.cc:1347] Generated new cluster ID: b6f6d707692b4e698dbd82dd444d20be
18:47:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.464264 20847 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
18:47:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.483357 20833 sys_catalog.cc:455] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a2b1eb1b95344d6b93c0721a08891cce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.484289 20833 sys_catalog.cc:458] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de [sys.catalog]: This master's current role is: FOLLOWER
18:47:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.517609 20847 catalog_manager.cc:1370] Generated new certificate authority record
18:47:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.520435 20847 catalog_manager.cc:1504] Loading token signing keys...
18:47:58.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.586629 20730 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } attempt: 7
18:47:58.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.618659 20847 catalog_manager.cc:5868] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce: Generated new TSK 0
18:47:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.619889 20847 catalog_manager.cc:1514] Initializing in-progress tserver states...
18:47:58.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.653398 20575 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/wal
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/data
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/logs
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/data/info.pb
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:47:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.193:0
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.23.193
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.193
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:47:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:47:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.656461 20730 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab: Bootstrap starting.
18:47:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.688650 20730 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab: Neither blocks nor log segments found. Creating new log.
18:47:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.692605 20730 log.cc:824] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab: Log is configured to *not* fsync() on all Append() calls
18:47:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.719000 20730 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab: No bootstrap required, opened a new log
18:47:58.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.801795 20730 raft_consensus.cc:348] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.804535 20730 raft_consensus.cc:374] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:47:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.805238 20730 raft_consensus.cc:729] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 892403f59bc14d3e9e75700ae62186ab, State: Initialized, Role: FOLLOWER
18:47:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.807631 20730 consensus_queue.cc:260] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } }
18:47:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.829607 20705 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2b1eb1b95344d6b93c0721a08891cce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "892403f59bc14d3e9e75700ae62186ab" is_pre_election: true
18:47:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.830329 20704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a2b1eb1b95344d6b93c0721a08891cce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "892403f59bc14d3e9e75700ae62186ab"
18:47:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.832157 20858 sys_catalog.cc:455] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.843778 20703 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [term 0 FOLLOWER]: Advancing to term 1
18:47:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.853243 20703 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [term 1 FOLLOWER]: Refusing update from remote peer a2b1eb1b95344d6b93c0721a08891cce: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:47:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.844909 20730 sys_catalog.cc:564] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [sys.catalog]: configured and running, proceeding with master startup.
18:47:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.855715 20839 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [LEADER]: Connected to new peer: Peer: permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:47:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.853960 20858 sys_catalog.cc:458] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [sys.catalog]: This master's current role is: FOLLOWER
18:47:58.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:58.889499 20684 tablet.cc:2369] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:47:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.933950 20858 sys_catalog.cc:455] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [sys.catalog]: SysCatalogTable state changed. Reason: New leader a2b1eb1b95344d6b93c0721a08891cce. Latest consensus state: current_term: 1 leader_uuid: "a2b1eb1b95344d6b93c0721a08891cce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.934762 20858 sys_catalog.cc:458] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [sys.catalog]: This master's current role is: FOLLOWER
18:47:58.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.936558 20864 sys_catalog.cc:455] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a2b1eb1b95344d6b93c0721a08891cce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.937750 20864 sys_catalog.cc:458] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab [sys.catalog]: This master's current role is: FOLLOWER
18:47:58.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:58.977969 20875 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
18:47:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:59.214174 20856 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:47:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:59.214860 20856 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:47:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:59.215654 20856 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:47:59.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:59.272539 20856 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:47:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:59.273912 20856 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.23.193
18:47:59.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.302162 20839 sys_catalog.cc:455] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a2b1eb1b95344d6b93c0721a08891cce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a2b1eb1b95344d6b93c0721a08891cce" member_type: VOTER last_known_addr { host: "127.20.23.254" port: 36031 } } peers { permanent_uuid: "892403f59bc14d3e9e75700ae62186ab" member_type: VOTER last_known_addr { host: "127.20.23.253" port: 44071 } } peers { permanent_uuid: "d0153a5e56be49749573e1d0648c15de" member_type: VOTER last_known_addr { host: "127.20.23.252" port: 40073 } } }
18:47:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.307011 20839 sys_catalog.cc:458] T 00000000000000000000000000000000 P a2b1eb1b95344d6b93c0721a08891cce [sys.catalog]: This master's current role is: LEADER
18:47:59.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.334857 20856 tablet_server_runner.cc:78] Tablet server non-default flags:
18:47:59.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:47:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:47:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:47:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:47:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/data
18:47:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/wal
18:47:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:47:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:47:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.193:0
18:47:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:47:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:47:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:47:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/data/info.pb
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.193
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.23.193
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/logs
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:47:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.337579 20856 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:47:59.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.340760 20856 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:47:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.357324 20848 catalog_manager.cc:1259] Loaded cluster ID: b6f6d707692b4e698dbd82dd444d20be
18:47:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.357596 20848 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de: loading cluster ID for follower catalog manager: success
18:47:59.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.365545 20848 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de: acquiring CA information for follower catalog manager: success
18:47:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:47:59.371582 20848 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d0153a5e56be49749573e1d0648c15de: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:47:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:59.386255 20884 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:00.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.007704 20875 catalog_manager.cc:1259] Loaded cluster ID: b6f6d707692b4e698dbd82dd444d20be
18:48:00.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.008085 20875 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab: loading cluster ID for follower catalog manager: success
18:48:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.045935 20875 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab: acquiring CA information for follower catalog manager: success
18:48:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.074982 20875 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 892403f59bc14d3e9e75700ae62186ab: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:48:00.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:47:59.395939 20885 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:00.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:00.778324 20883 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 20856
18:48:00.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:00.908530 20856 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.521s user 0.557s sys 0.696s
18:48:00.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:00.909674 20856 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.522s user 0.558s sys 0.696s
18:48:00.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.909951 20856 server_base.cc:1034] running on GCE node
18:48:00.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:00.911880 20887 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.916101 20856 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:00.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.930761 20856 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:00.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.934437 20856 hybrid_clock.cc:648] HybridClock initialized: now 1781894880934351 us; error 100 us; skew 500 ppm
18:48:00.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.935572 20856 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.969321 20856 webserver.cc:460] Webserver started at http://127.20.23.193:46475/ using document root <none> and password file <none>
18:48:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.970679 20856 fs_manager.cc:359] Metadata directory not provided
18:48:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.970966 20856 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.971498 20856 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:48:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.978770 20856 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/data/instance:
18:48:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b62bfe32a0664054affad1b37b030da1"
18:48:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:00 on dist-test-slave-9qt6"
18:48:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:00.980438 20856 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/wal/instance:
18:48:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b62bfe32a0664054affad1b37b030da1"
18:48:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:00 on dist-test-slave-9qt6"
18:48:01.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.000969 20856 fs_manager.cc:691] Time spent creating directory manager: real 0.020s user 0.012s sys 0.008s
18:48:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.031394 20895 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:01.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.033179 20856 fs_manager.cc:725] Time spent opening block manager: real 0.025s user 0.005s sys 0.019s
18:48:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.033952 20856 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/wal
18:48:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b62bfe32a0664054affad1b37b030da1"
18:48:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:00 on dist-test-slave-9qt6"
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.034564 20856 fs_report.cc:383] FS layout report
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/wal
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/wal
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/data/data
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:01.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.133667 20856 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:01.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.136497 20856 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:01.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.137233 20856 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.148146 20856 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:48:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.161520 20856 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:48:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.161901 20856 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
18:48:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.162288 20856 ts_tablet_manager.cc:610] Registered 0 tablets
18:48:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.162503 20856 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
18:48:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.601557 20856 rpc_server.cc:307] RPC server started. Bound to: 127.20.23.193:43773
18:48:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.601732 21007 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.23.193:43773 every 8 connection(s)
18:48:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.606984 20856 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/data/info.pb
18:48:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.610780 20575 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 20856
18:48:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.611243 20575 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-0/wal/instance
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.624966 20575 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/wal
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/data
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/logs
18:48:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/data/info.pb
18:48:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.194:0
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.23.194
18:48:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.194
18:48:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:48:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:48:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:48:01.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.710580 21008 heartbeater.cc:347] Connected to a master server at 127.20.23.253:44071
18:48:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.711123 21008 heartbeater.cc:464] Registering TS with master...
18:48:01.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.712567 21008 heartbeater.cc:511] Master 127.20.23.253:44071 requested a full tablet report, sending...
18:48:01.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.717053 20695 ts_manager.cc:194] Registered new tserver with Master: b62bfe32a0664054affad1b37b030da1 (127.20.23.193:43773)
18:48:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.722707 21012 heartbeater.cc:347] Connected to a master server at 127.20.23.254:36031
18:48:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.723232 21012 heartbeater.cc:464] Registering TS with master...
18:48:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.724290 21012 heartbeater.cc:511] Master 127.20.23.254:36031 requested a full tablet report, sending...
18:48:01.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.725479 21009 heartbeater.cc:347] Connected to a master server at 127.20.23.252:40073
18:48:01.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.725939 21009 heartbeater.cc:464] Registering TS with master...
18:48:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.727914 21009 heartbeater.cc:511] Master 127.20.23.252:40073 requested a full tablet report, sending...
18:48:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.732642 20620 ts_manager.cc:194] Registered new tserver with Master: b62bfe32a0664054affad1b37b030da1 (127.20.23.193:43773)
18:48:01.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.737802 20774 ts_manager.cc:194] Registered new tserver with Master: b62bfe32a0664054affad1b37b030da1 (127.20.23.193:43773)
18:48:01.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:01.737946 20620 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.23.193:53383
18:48:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:02.129999 21017 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:48:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:02.130511 21017 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:48:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:02.130963 21017 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:48:02.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:02.169215 21017 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:48:02.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:02.170298 21017 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.23.194
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:02.212441 21017 tablet_server_runner.cc:78] Tablet server non-default flags:
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/data
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/wal
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.194:0
18:48:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/data/info.pb
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.194
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.23.194
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/logs
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:48:02.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:48:02.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:48:02.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:48:02.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:48:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:48:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:48:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:48:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:48:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:02.214632 21017 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:48:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:02.217660 21017 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:48:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:02.257889 21025 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:02.747949 21012 heartbeater.cc:503] Master 127.20.23.254:36031 was elected leader, sending a full tablet report...
18:48:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:02.258733 21026 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:03.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:03.632509 21028 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:03.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:03.636262 21027 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1375 milliseconds
18:48:03.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.636507 21017 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:48:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.637987 21017 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:03.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.640601 21017 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:03.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.642082 21017 hybrid_clock.cc:648] HybridClock initialized: now 1781894883642022 us; error 37 us; skew 500 ppm
18:48:03.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.643121 21017 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:03.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.651067 21017 webserver.cc:460] Webserver started at http://127.20.23.194:34461/ using document root <none> and password file <none>
18:48:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.652297 21017 fs_manager.cc:359] Metadata directory not provided
18:48:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.652566 21017 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.653335 21017 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:48:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.659950 21017 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/data/instance:
18:48:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fadbc561508945ca9c5b644929f9e5b7"
18:48:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:03 on dist-test-slave-9qt6"
18:48:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.661685 21017 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/wal/instance:
18:48:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fadbc561508945ca9c5b644929f9e5b7"
18:48:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:03 on dist-test-slave-9qt6"
18:48:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.671389 21017 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
18:48:03.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.678203 21043 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:03.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.679553 21017 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
18:48:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.679978 21017 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/wal
18:48:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fadbc561508945ca9c5b644929f9e5b7"
18:48:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:03 on dist-test-slave-9qt6"
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.680514 21017 fs_report.cc:383] FS layout report
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/wal
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/wal
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/data/data
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.724907 21017 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.726743 21017 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.727321 21017 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.730731 21017 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:48:03.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.735579 21017 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:48:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.735831 21017 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
18:48:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.736168 21017 ts_tablet_manager.cc:610] Registered 0 tablets
18:48:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.736465 21017 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
18:48:03.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.928122 21017 rpc_server.cc:307] RPC server started. Bound to: 127.20.23.194:37683
18:48:03.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.928249 21155 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.23.194:37683 every 8 connection(s)
18:48:03.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.930938 21017 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/data/info.pb
18:48:03.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.934778 20575 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 21017
18:48:03.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.935572 20575 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-1/wal/instance
18:48:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.955399 20575 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/wal
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/data
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/logs
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/data/info.pb
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.195:0
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.23.195
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.195
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:48:03.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.997747 21156 heartbeater.cc:347] Connected to a master server at 127.20.23.253:44071
18:48:03.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.998430 21156 heartbeater.cc:464] Registering TS with master...
18:48:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:03.999856 21156 heartbeater.cc:511] Master 127.20.23.253:44071 requested a full tablet report, sending...
18:48:04.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.004520 20695 ts_manager.cc:194] Registered new tserver with Master: fadbc561508945ca9c5b644929f9e5b7 (127.20.23.194:37683)
18:48:04.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.006009 21157 heartbeater.cc:347] Connected to a master server at 127.20.23.252:40073
18:48:04.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.006438 21157 heartbeater.cc:464] Registering TS with master...
18:48:04.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.007292 21157 heartbeater.cc:511] Master 127.20.23.252:40073 requested a full tablet report, sending...
18:48:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.009781 21158 heartbeater.cc:347] Connected to a master server at 127.20.23.254:36031
18:48:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.010154 21158 heartbeater.cc:464] Registering TS with master...
18:48:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.009938 20774 ts_manager.cc:194] Registered new tserver with Master: fadbc561508945ca9c5b644929f9e5b7 (127.20.23.194:37683)
18:48:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.011104 21158 heartbeater.cc:511] Master 127.20.23.254:36031 requested a full tablet report, sending...
18:48:04.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.013921 20620 ts_manager.cc:194] Registered new tserver with Master: fadbc561508945ca9c5b644929f9e5b7 (127.20.23.194:37683)
18:48:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.015697 20620 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.23.194:38269
18:48:04.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:04.301275 21163 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:48:04.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:04.301878 21163 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:48:04.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:04.302552 21163 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:48:04.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:04.336033 21163 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:48:04.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:04.336977 21163 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.23.195
18:48:04.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.371402 21163 tablet_server_runner.cc:78] Tablet server non-default flags:
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.23.212:40225
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/data
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/wal
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.23.195:0
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/data/info.pb
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.23.195
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.23.254:36031,127.20.23.253:44071,127.20.23.252:40073
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.23.195
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/logs
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:48:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:48:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:48:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:48:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:48:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:48:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:48:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:48:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:48:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.372665 21163 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:48:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.374215 21163 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:48:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:04.396751 21172 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:05.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.020226 21158 heartbeater.cc:503] Master 127.20.23.254:36031 was elected leader, sending a full tablet report...
18:48:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:04.396991 21173 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:05.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:04.398875 21175 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:05.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:04.400580 21163 server_base.cc:1034] running on GCE node
18:48:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.509258 21163 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.512593 21163 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.514221 21163 hybrid_clock.cc:648] HybridClock initialized: now 1781894885514128 us; error 89 us; skew 500 ppm
18:48:05.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.515137 21163 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:05.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.522455 21163 webserver.cc:460] Webserver started at http://127.20.23.195:39123/ using document root <none> and password file <none>
18:48:05.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.523663 21163 fs_manager.cc:359] Metadata directory not provided
18:48:05.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.523923 21163 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:05.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.524483 21163 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:48:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.530509 21163 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/data/instance:
18:48:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b31ea32c9c34b66847fd588c8a2c441"
18:48:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:05 on dist-test-slave-9qt6"
18:48:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.532196 21163 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/wal/instance:
18:48:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b31ea32c9c34b66847fd588c8a2c441"
18:48:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:05 on dist-test-slave-9qt6"
18:48:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.541188 21163 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
18:48:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.547959 21183 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.549319 21163 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.549700 21163 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/wal
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b31ea32c9c34b66847fd588c8a2c441"
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:05 on dist-test-slave-9qt6"
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.550069 21163 fs_report.cc:383] FS layout report
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/wal
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/wal
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/data/data
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:05.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.591216 21163 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.592965 21163 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.593525 21163 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.596976 21163 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:48:05.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.602411 21163 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:48:05.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.602833 21163 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
18:48:05.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.603283 21163 ts_tablet_manager.cc:610] Registered 0 tablets
18:48:05.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.603523 21163 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
18:48:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.802381 21163 rpc_server.cc:307] RPC server started. Bound to: 127.20.23.195:43215
18:48:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.802619 21295 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.23.195:43215 every 8 connection(s)
18:48:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.805406 21163 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/data/info.pb
18:48:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.814728 20575 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 21163
18:48:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.815194 20575 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster5742209250075282745/ts-2/wal/instance
18:48:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.862538 21299 heartbeater.cc:347] Connected to a master server at 127.20.23.254:36031
18:48:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.863106 21299 heartbeater.cc:464] Registering TS with master...
18:48:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.864521 21299 heartbeater.cc:511] Master 127.20.23.254:36031 requested a full tablet report, sending...
18:48:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.867115 21296 heartbeater.cc:347] Connected to a master server at 127.20.23.253:44071
18:48:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.867334 21297 heartbeater.cc:347] Connected to a master server at 127.20.23.252:40073
18:48:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.867478 21296 heartbeater.cc:464] Registering TS with master...
18:48:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.867651 21297 heartbeater.cc:464] Registering TS with master...
18:48:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.867856 20620 ts_manager.cc:194] Registered new tserver with Master: 0b31ea32c9c34b66847fd588c8a2c441 (127.20.23.195:43215)
18:48:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.868490 21297 heartbeater.cc:511] Master 127.20.23.252:40073 requested a full tablet report, sending...
18:48:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.868487 21296 heartbeater.cc:511] Master 127.20.23.253:44071 requested a full tablet report, sending...
18:48:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.869750 20620 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.23.195:45079
18:48:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.872365 20774 ts_manager.cc:194] Registered new tserver with Master: 0b31ea32c9c34b66847fd588c8a2c441 (127.20.23.195:43215)
18:48:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.872921 20694 ts_manager.cc:194] Registered new tserver with Master: 0b31ea32c9c34b66847fd588c8a2c441 (127.20.23.195:43215)
18:48:05.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:05.895274 20575 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
18:48:05.908 [DEBUG - main] (MiniKuduCluster.java:195) Response:
18:48:05.909 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
18:48:05.920 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.23.254"
port: 36031
}
bound_http_address {
host: "127.20.23.254"
port: 42731
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.23.253"
port: 44071
}
bound_http_address {
host: "127.20.23.253"
port: 44055
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.23.252"
port: 40073
}
bound_http_address {
host: "127.20.23.252"
port: 38771
}
}
}
18:48:05.927 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
18:48:05.931 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.20.23.193"
port: 43773
}
bound_http_address {
host: "127.20.23.193"
port: 46475
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.20.23.194"
port: 37683
}
bound_http_address {
host: "127.20.23.194"
port: 34461
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.20.23.195"
port: 43215
}
bound_http_address {
host: "127.20.23.195"
port: 39123
}
}
}
18:48:05.931 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
18:48:06.122 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
18:48:06.273 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskvcoJf5/test-tmp
18:48:06.274 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
18:48:06.275 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskvcoJf5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
18:48:06.279 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
18:48:06.705 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:06.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
18:48:06.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:06.710697 21307 mini_oidc.cc:152] Starting JWKS server
18:48:06.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:06.715453 21307 webserver.cc:460] Webserver started at http://127.0.0.1:36641/ using document root <none> and password file <none>
18:48:06.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:06.715970 21307 mini_oidc.cc:189] Starting OIDC Discovery server
18:48:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:06.717995 21307 webserver.cc:460] Webserver started at http://127.0.0.1:34689/ using document root <none> and password file <none>
18:48:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
18:48:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/principal' for realm 'KRBTEST.COM',
18:48:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
18:48:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:06 dist-test-slave-9qt6 krb5kdc[21320](info): setting up network...
18:48:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
18:48:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:06 dist-test-slave-9qt6 krb5kdc[21320](info): set up 2 sockets
18:48:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:06 dist-test-slave-9qt6 krb5kdc[21320](info): commencing operation
18:48:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
18:48:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:06.877452 21299 heartbeater.cc:503] Master 127.20.23.254:36031 was elected leader, sending a full tablet report...
18:48:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:07.787637 21307 mini_kdc.cc:121] Time spent starting KDC: real 1.041s user 0.002s sys 0.026s
18:48:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
18:48:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
18:48:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
18:48:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
18:48:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
18:48:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
18:48:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
18:48:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
18:48:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
18:48:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
18:48:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/test-user.keytab.
18:48:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/test-user.keytab.
18:48:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:07 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894887, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
18:48:07.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:48:07Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
18:48:07.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:48:07Z Disabled control of system clock
18:48:07.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.206.254@KRBTEST.COM; defaulting to no policy
18:48:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:48:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.206.254@KRBTEST.COM" created.
18:48:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:48:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.206.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.254.keytab.
18:48:07.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.206.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.254.keytab.
18:48:08.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:08.000257 21307 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:08.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/logs
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/info.pb
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.254:39439
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.254
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.254.keytab
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.254
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:48:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:48:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.318085 21333 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:48:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.318651 21333 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:48:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.319442 21333 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:48:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.352561 21333 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:48:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.352802 21333 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:48:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.353063 21333 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:48:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.353258 21333 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:48:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.353430 21333 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:48:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.353596 21333 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:48:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.353721 21333 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36641/jwks.json
18:48:08.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:08.390285 21333 master_runner.cc:386] Master server non-default flags:
18:48:08.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:08.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.254.keytab
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.254
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.254:39439
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/info.pb
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.254
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/logs
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:48:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:08.391485 21333 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:48:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:08.392653 21333 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:48:08.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.416947 21342 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.418452 21340 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:08.416963 21339 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:08.419242 21333 server_base.cc:1034] running on GCE node
18:48:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.551740 21333 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.555761 21333 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.557343 21333 hybrid_clock.cc:648] HybridClock initialized: now 1781894889557279 us; error 89 us; skew 500 ppm
18:48:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.558408 21333 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:09 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894889, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.566905 21333 init.cc:377] Logged in from keytab as kudu/127.20.206.254@KRBTEST.COM (short username kudu)
18:48:09.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.575210 21333 webserver.cc:460] Webserver started at http://127.20.206.254:38837/ using document root <none> and password file <none>
18:48:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.576412 21333 fs_manager.cc:359] Metadata directory not provided
18:48:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.576646 21333 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.577111 21333 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:48:09.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.583101 21333 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/instance:
18:48:09.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "281d7f64b5fd4edeb2e8d3c6c469300b"
18:48:09.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:09 on dist-test-slave-9qt6"
18:48:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.584282 21333 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal/instance:
18:48:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "281d7f64b5fd4edeb2e8d3c6c469300b"
18:48:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:09 on dist-test-slave-9qt6"
18:48:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.594575 21333 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
18:48:09.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.604775 21351 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:09.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.606382 21333 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.006s sys 0.001s
18:48:09.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.606879 21333 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:09.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "281d7f64b5fd4edeb2e8d3c6c469300b"
18:48:09.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:09 on dist-test-slave-9qt6"
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.607381 21333 fs_report.cc:383] FS layout report
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/data
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.666344 21333 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.677281 21333 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.678332 21333 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.786388 21333 rpc_server.cc:307] RPC server started. Bound to: 127.20.206.254:39439
18:48:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.786451 21402 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.206.254:39439 every 8 connection(s)
18:48:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.789637 21333 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/info.pb
18:48:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.796779 21307 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 21333
18:48:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.797199 21403 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:48:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.797315 21307 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal/instance
18:48:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.808948 21403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } has no permanent_uuid. Determining permanent_uuid...
18:48:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.206.253@KRBTEST.COM; defaulting to no policy
18:48:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:48:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.206.253@KRBTEST.COM" created.
18:48:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:09 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894889, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.254@KRBTEST.COM for kudu/127.20.206.254@KRBTEST.COM
18:48:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:48:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.206.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.253.keytab.
18:48:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.206.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.253.keytab.
18:48:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.849574 21403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.851101 21307 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/logs
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/info.pb
18:48:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.253:44071
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.253
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.253.keytab
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.253
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:48:09.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:09.852815 21353 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:09.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:09.857626 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.907231 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } attempt: 1
18:48:09.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:09.911939 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:09.980305 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } attempt: 2
18:48:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:09.984285 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:10.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:10.075644 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } attempt: 3
18:48:10.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.079876 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:10.223294 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } attempt: 4
18:48:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.228497 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:10.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.249588 21410 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:48:10.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.250221 21410 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:48:10.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.250813 21410 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:48:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.282692 21410 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:48:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.282934 21410 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:48:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.283186 21410 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:48:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.283378 21410 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:48:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.283564 21410 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:48:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.283751 21410 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:48:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.283887 21410 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36641/jwks.json
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:10.320099 21410 master_runner.cc:386] Master server non-default flags:
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.253.keytab
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.253
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.253:44071
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/info.pb
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.253
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/logs
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:48:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:48:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:48:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:10.321606 21410 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:48:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:10.322827 21410 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:48:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.342690 21417 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:10.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:10.527933 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } attempt: 5
18:48:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.531137 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.688704 21292 debug-util.cc:398] Leaking SignalData structure 0x7b08000afa60 after lost signal to thread 21167
18:48:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.690569 21292 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba200 after lost signal to thread 21295
18:48:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.078605 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } attempt: 6
18:48:11.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:11.090361 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.253:44071: Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:11.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.343343 21420 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:11.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:10.342860 21418 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:11.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:11.544628 21419 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
18:48:11.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.544821 21410 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:48:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.549252 21410 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.553354 21410 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:11.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.555245 21410 hybrid_clock.cc:648] HybridClock initialized: now 1781894891555192 us; error 34 us; skew 500 ppm
18:48:11.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.556524 21410 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:11 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894891, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:11.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.568847 21410 init.cc:377] Logged in from keytab as kudu/127.20.206.253@KRBTEST.COM (short username kudu)
18:48:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.576046 21410 webserver.cc:460] Webserver started at http://127.20.206.253:32921/ using document root <none> and password file <none>
18:48:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.577502 21410 fs_manager.cc:359] Metadata directory not provided
18:48:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.577818 21410 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.578476 21410 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:48:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.586755 21410 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/instance:
18:48:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93485ec69e05410b8daf427106313eea"
18:48:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:11 on dist-test-slave-9qt6"
18:48:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.588841 21410 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal/instance:
18:48:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93485ec69e05410b8daf427106313eea"
18:48:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:11 on dist-test-slave-9qt6"
18:48:11.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.604444 21410 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.010s sys 0.002s
18:48:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.613646 21431 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:11.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.615468 21410 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.007s sys 0.000s
18:48:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.616127 21410 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93485ec69e05410b8daf427106313eea"
18:48:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:11 on dist-test-slave-9qt6"
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.616710 21410 fs_report.cc:383] FS layout report
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/data
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:11.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.671244 21410 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:11.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.680792 21410 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:11.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.681456 21410 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:11.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.815807 21482 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.206.253:44071 every 8 connection(s)
18:48:11.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.815829 21410 rpc_server.cc:307] RPC server started. Bound to: 127.20.206.253:44071
18:48:11.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.820160 21410 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/info.pb
18:48:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.821067 21307 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 21410
18:48:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.821470 21307 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal/instance
18:48:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.828415 21483 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:48:11.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.206.252@KRBTEST.COM; defaulting to no policy
18:48:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:48:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.206.252@KRBTEST.COM" created.
18:48:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.845685 21483 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } has no permanent_uuid. Determining permanent_uuid...
18:48:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:48:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.206.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.252.keytab.
18:48:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.206.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.252.keytab.
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.876072 21307 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/logs
18:48:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/info.pb
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.252:35459
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.252
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.252.keytab
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.252
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:48:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:11 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894891, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.253@KRBTEST.COM for kudu/127.20.206.254@KRBTEST.COM
18:48:11.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.902673 21483 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
18:48:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:11 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894891, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.253@KRBTEST.COM for kudu/127.20.206.253@KRBTEST.COM
18:48:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:11.943490 21483 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } has no permanent_uuid. Determining permanent_uuid...
18:48:11.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:11.946727 21435 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:11.952838 21483 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.002475 21483 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 1
18:48:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.007341 21483 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.075796 21483 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 2
18:48:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.080855 21483 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.150872 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } attempt: 7
18:48:12.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:12 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894889, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.254@KRBTEST.COM for kudu/127.20.206.253@KRBTEST.COM
18:48:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.172387 21483 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 3
18:48:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.180707 21483 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.184235 21403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } has no permanent_uuid. Determining permanent_uuid...
18:48:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.190702 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.241325 20726 messenger.cc:188] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.20.23.253:44071
18:48:12.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.249162 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 1
18:48:12.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.254142 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.281869 21489 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:48:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.282503 21489 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:48:12.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.283063 21489 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:48:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.315274 21489 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:48:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.315491 21489 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:48:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.315742 21489 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:48:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.315922 21489 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:48:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.316097 21489 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:48:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.316293 21489 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:48:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.316433 21489 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36641/jwks.json
18:48:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.324187 21483 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 4
18:48:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.329011 21483 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.335608 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 2
18:48:12.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.340169 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.358798 21489 master_runner.cc:386] Master server non-default flags:
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.252.keytab
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.252
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.252:35459
18:48:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/info.pb
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.252
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/logs
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:48:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:48:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:48:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:48:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:48:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:48:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.360038 21489 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:48:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.361019 21489 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:48:12.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.382261 21500 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.425614 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 3
18:48:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.432271 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.572779 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 4
18:48:12.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.576613 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.628537 21483 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 5
18:48:12.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.633651 21483 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:12.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.842831 21479 messenger.cc:188] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.20.206.253:44071
18:48:12.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:12.860235 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 5
18:48:12.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.864995 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:13.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.181145 21483 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 6
18:48:13.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:13.189464 21483 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.417594 21403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } attempt: 6
18:48:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:13.425594 21403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.206.252:35459: Network error: Client connection negotiation failed: client connection to 127.20.206.252:35459: connect: Connection refused (error 111)
18:48:13.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.382323 21503 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:13.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:12.385563 21501 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:13.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:13.555648 21502 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
18:48:13.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.555796 21489 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:48:13.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.559712 21489 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:13.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.562427 21489 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:13.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.563872 21489 hybrid_clock.cc:648] HybridClock initialized: now 1781894893563841 us; error 52 us; skew 500 ppm
18:48:13.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.564889 21489 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:13 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894893, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:13.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.570834 21489 init.cc:377] Logged in from keytab as kudu/127.20.206.252@KRBTEST.COM (short username kudu)
18:48:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.576987 21489 webserver.cc:460] Webserver started at http://127.20.206.252:36429/ using document root <none> and password file <none>
18:48:13.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.577967 21489 fs_manager.cc:359] Metadata directory not provided
18:48:13.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.578210 21489 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:13.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.578704 21489 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:48:13.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.583616 21489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/instance:
18:48:13.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "09106035e7bf4087b858c3eb5f795743"
18:48:13.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:13 on dist-test-slave-9qt6"
18:48:13.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.584790 21489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal/instance:
18:48:13.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "09106035e7bf4087b858c3eb5f795743"
18:48:13.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:13 on dist-test-slave-9qt6"
18:48:13.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.593705 21489 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.002s sys 0.007s
18:48:13.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.600811 21517 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.603161 21489 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
18:48:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.603502 21489 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "09106035e7bf4087b858c3eb5f795743"
18:48:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:13 on dist-test-slave-9qt6"
18:48:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.603852 21489 fs_report.cc:383] FS layout report
18:48:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/data
18:48:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.650290 21489 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:13.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.658591 21489 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:13.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.659168 21489 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.758165 21489 rpc_server.cc:307] RPC server started. Bound to: 127.20.206.252:35459
18:48:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.758211 21568 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.206.252:35459 every 8 connection(s)
18:48:13.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.761432 21489 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/info.pb
18:48:13.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.765446 21307 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 21489
18:48:13.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.766003 21307 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal/instance
18:48:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.769764 21569 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:48:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.787495 21569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } has no permanent_uuid. Determining permanent_uuid...
18:48:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:13 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894887, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.206.252@KRBTEST.COM
18:48:13.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:13 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894893, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.252@KRBTEST.COM for kudu/127.20.206.254@KRBTEST.COM
18:48:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.836333 21569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
18:48:13.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:13 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894893, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.252@KRBTEST.COM for kudu/127.20.206.253@KRBTEST.COM
18:48:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:13 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894887, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.206.253@KRBTEST.COM
18:48:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.870529 21569 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } has no permanent_uuid. Determining permanent_uuid...
18:48:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:13 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894893, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.252@KRBTEST.COM for kudu/127.20.206.252@KRBTEST.COM
18:48:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:13 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894887, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.206.254@KRBTEST.COM
18:48:13.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.906842 21569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: Bootstrap starting.
18:48:13.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.909541 21307 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
18:48:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.916924 21569 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: Neither blocks nor log segments found. Creating new log.
18:48:13.917 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:13.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
18:48:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.919535 21569 log.cc:824] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: Log is configured to *not* fsync() on all Append() calls
18:48:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.926555 21569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: No bootstrap required, opened a new log
18:48:13.937 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
18:48:13.938 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
18:48:13.943 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.206.254"
port: 39439
}
bound_http_address {
host: "127.20.206.254"
port: 38837
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.206.253"
port: 44071
}
bound_http_address {
host: "127.20.206.253"
port: 32921
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.206.252"
port: 35459
}
bound_http_address {
host: "127.20.206.252"
port: 36429
}
}
}
18:48:13.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
18:48:13.946 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:13.946 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
18:48:13.948 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
18:48:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.960924 21569 raft_consensus.cc:348] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.961725 21569 raft_consensus.cc:374] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:48:13.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.962041 21569 raft_consensus.cc:729] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 09106035e7bf4087b858c3eb5f795743, State: Initialized, Role: FOLLOWER
18:48:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.963289 21569 consensus_queue.cc:260] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.967276 21579 sys_catalog.cc:455] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } } }
18:48:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.968327 21579 sys_catalog.cc:458] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [sys.catalog]: This master's current role is: FOLLOWER
18:48:13.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:13.969463 21569 sys_catalog.cc:564] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [sys.catalog]: configured and running, proceeding with master startup.
18:48:13.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:13 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894893, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:13.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
18:48:13.983 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:13.986 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc
18:48:13.995 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
18:48:13.996 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc
18:48:14.000 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
18:48:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.007593 21591 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.008596 21591 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:14.012 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
18:48:14.017 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
18:48:14.017 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
18:48:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.004246 21489 master_runner.cc:186] Error getting master registration for 127.20.206.254:39439: OK, instance_id { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" instance_seqno: 1781894889755986 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:48:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.040951 21489 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:48:14.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:14 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894894, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:14.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
18:48:14.063 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:14.063 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc
18:48:14.065 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
18:48:14.066 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.206.253:44071
18:48:14.068 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
18:48:14.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.068851 21307 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 21410
18:48:14.094 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:14.095 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.206.252:35459
18:48:14.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
18:48:14.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.095947 21307 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 21489
18:48:14.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:14.122 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.206.254:39439
18:48:14.122 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
18:48:14.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.122730 21307 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 21333
18:48:14.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:14.144 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.206.253:44071
18:48:14.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
18:48:14.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.148178 21307 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:14.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:14.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:14.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/logs
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/info.pb
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.253:44071
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.253
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32921
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.253.keytab
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.253
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:48:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.470258 21595 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:48:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.470849 21595 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:48:14.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.471511 21595 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:48:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.512606 21595 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:48:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.512885 21595 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:48:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.513208 21595 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:48:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.513438 21595 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:48:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.513664 21595 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:48:14.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.513873 21595 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:48:14.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.514019 21595 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36641/jwks.json
18:48:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.549850 21595 master_runner.cc:386] Master server non-default flags:
18:48:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.253.keytab
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.253
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.253:44071
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:48:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/info.pb
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.253
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32921
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/logs
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.551322 21595 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:48:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.552609 21595 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:48:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.572086 21603 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.572583 21605 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:14.574621 21595 server_base.cc:1034] running on GCE node
18:48:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:14.572243 21602 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.717386 21595 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.720463 21595 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.721917 21595 hybrid_clock.cc:648] HybridClock initialized: now 1781894895721841 us; error 95 us; skew 500 ppm
18:48:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.722754 21595 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:15 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894895, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.729494 21595 init.cc:377] Logged in from keytab as kudu/127.20.206.253@KRBTEST.COM (short username kudu)
18:48:15.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.734767 21595 webserver.cc:460] Webserver started at http://127.20.206.253:32921/ using document root <none> and password file <none>
18:48:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.735731 21595 fs_manager.cc:359] Metadata directory not provided
18:48:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.736008 21595 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:15.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.744621 21595 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
18:48:15.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.749467 21614 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:15.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.750921 21595 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
18:48:15.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.751302 21595 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93485ec69e05410b8daf427106313eea"
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:11 on dist-test-slave-9qt6"
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.753724 21595 fs_report.cc:383] FS layout report
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/data
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:15.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.795333 21595 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:15.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.801892 21595 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:15.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.802459 21595 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.886261 21595 rpc_server.cc:307] RPC server started. Bound to: 127.20.206.253:44071
18:48:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.886445 21665 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.206.253:44071 every 8 connection(s)
18:48:15.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.889117 21595 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/data/info.pb
18:48:15.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.890226 21307 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 21595
18:48:15.891 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:15.891 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.206.252:35459
18:48:15.891 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
18:48:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.893414 21307 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data
18:48:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/logs
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/info.pb
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.252:35459
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.252
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36429
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.252.keytab
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.252
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:15.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:15.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:15.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:15.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:15.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:15.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:48:15.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.907614 21666 sys_catalog.cc:263] Verifying existing consensus state
18:48:15.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 18:48:15.908970 21666 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
18:48:15.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.909600 21595 master.cc:537] Master@127.20.206.253:44071 shutting down...
18:48:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:15.983893 21595 master.cc:552] Master@127.20.206.253:44071 shutdown complete.
18:48:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
18:48:16.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.228188 21668 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:48:16.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.228946 21668 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:48:16.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.229697 21668 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:48:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.271855 21668 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:48:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.272146 21668 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:48:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.272553 21668 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:48:16.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.272890 21668 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:48:16.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.273204 21668 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:48:16.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.273504 21668 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:48:16.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.273738 21668 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36641/jwks.json
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:16.311477 21668 master_runner.cc:386] Master server non-default flags:
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.252.keytab
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.252
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.252:35459
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/info.pb
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.252
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36429
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/logs
18:48:16.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:16.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:16.312705 21668 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:48:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:16.313795 21668 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:48:16.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.333642 21674 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:17.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.333760 21675 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:16.334023 21677 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:17.483058 21676 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
18:48:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.483222 21668 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:48:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.488215 21668 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.491210 21668 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.492679 21668 hybrid_clock.cc:648] HybridClock initialized: now 1781894897492629 us; error 88 us; skew 500 ppm
18:48:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.493628 21668 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:17.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:17 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894897, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:17.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.503096 21668 init.cc:377] Logged in from keytab as kudu/127.20.206.252@KRBTEST.COM (short username kudu)
18:48:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.508708 21668 webserver.cc:460] Webserver started at http://127.20.206.252:36429/ using document root <none> and password file <none>
18:48:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.509593 21668 fs_manager.cc:359] Metadata directory not provided
18:48:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.509799 21668 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.517982 21668 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
18:48:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.522783 21687 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:17.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.523849 21668 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.524159 21668 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "09106035e7bf4087b858c3eb5f795743"
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:13 on dist-test-slave-9qt6"
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.526922 21668 fs_report.cc:383] FS layout report
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/wal
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/data
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:17.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:17.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:17.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.562752 21668 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.568571 21668 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.569001 21668 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.644097 21668 rpc_server.cc:307] RPC server started. Bound to: 127.20.206.252:35459
18:48:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.644272 21738 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.206.252:35459 every 8 connection(s)
18:48:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.646919 21668 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-2/data/info.pb
18:48:17.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.654825 21307 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 21668
18:48:17.656 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:17.656 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.206.254:39439
18:48:17.656 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
18:48:17.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.658145 21307 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:17.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu
18:48:17.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/logs
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/info.pb
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.254:39439
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.254
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38837
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.254.keytab
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.254
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:48:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.659811 21739 sys_catalog.cc:263] Verifying existing consensus state
18:48:17.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.667742 21739 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: Bootstrap starting.
18:48:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.717978 21739 log.cc:824] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: Log is configured to *not* fsync() on all Append() calls
18:48:17.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.723464 21739 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:48:17.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.724225 21739 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: Bootstrap complete.
18:48:17.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.753204 21739 raft_consensus.cc:348] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.757797 21739 raft_consensus.cc:374] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:48:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.758157 21739 raft_consensus.cc:729] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 09106035e7bf4087b858c3eb5f795743, State: Initialized, Role: FOLLOWER
18:48:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.759269 21739 consensus_queue.cc:260] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.762610 21743 sys_catalog.cc:455] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } } }
18:48:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.763621 21743 sys_catalog.cc:458] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [sys.catalog]: This master's current role is: FOLLOWER
18:48:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.764497 21739 sys_catalog.cc:564] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [sys.catalog]: configured and running, proceeding with master startup.
18:48:17.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:17.801275 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:17.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:17.801668 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:17.824349 21693 tablet.cc:2369] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:48:17.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.801412 21668 master_runner.cc:186] Error getting master registration for 127.20.206.254:39439: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111),
18:48:17.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:17.835404 21668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:48:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.009150 21741 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:48:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.009780 21741 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:48:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.010639 21741 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:48:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.042934 21741 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:48:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.043203 21741 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:48:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.043601 21741 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:48:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.043905 21741 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:48:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.044198 21741 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:48:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.044533 21741 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:48:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.044734 21741 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36641/jwks.json
18:48:18.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:18.099363 21741 master_runner.cc:386] Master server non-default flags:
18:48:18.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.206.212:41175
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.206.254:39439,127.20.206.253:44071,127.20.206.252:35459
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/krb5kdc/kudu_127.20.206.254.keytab
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.206.254
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.206.254:39439
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:48:18.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36641/jwks.json
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/info.pb
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.206.254
18:48:18.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38837
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/logs
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:18.100857 21741 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:48:18.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:18.102289 21741 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:48:18.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:18.118883 21743 raft_consensus.cc:482] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:48:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:18.119509 21743 raft_consensus.cc:504] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:18.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:18.125481 21743 leader_election.cc:288] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439), 93485ec69e05410b8daf427106313eea (127.20.206.253:44071)
18:48:18.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.125481 21690 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111)
18:48:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.134989 21763 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:18.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.142170 21690 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439): Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111)
18:48:18.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.143304 21689 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 93485ec69e05410b8daf427106313eea (127.20.206.253:44071): Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:18.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:18.144006 21689 leader_election.cc:302] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 09106035e7bf4087b858c3eb5f795743; no voters: 281d7f64b5fd4edeb2e8d3c6c469300b, 93485ec69e05410b8daf427106313eea
18:48:18.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:18.145283 21743 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:48:18.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.806756 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:18.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.807453 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:19.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.135867 21762 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:19.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:18.146467 21765 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:48:19.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:19.488085 21764 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1362 milliseconds
18:48:19.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.488276 21741 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:48:19.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.490660 21741 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:48:19.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.494261 21741 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:48:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.497848 21741 hybrid_clock.cc:648] HybridClock initialized: now 1781894899497797 us; error 77 us; skew 500 ppm
18:48:19.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.498983 21741 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:48:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:19 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894899, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:19.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.506943 21741 init.cc:377] Logged in from keytab as kudu/127.20.206.254@KRBTEST.COM (short username kudu)
18:48:19.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.521553 21741 webserver.cc:460] Webserver started at http://127.20.206.254:38837/ using document root <none> and password file <none>
18:48:19.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.522876 21741 fs_manager.cc:359] Metadata directory not provided
18:48:19.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.523175 21741 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:48:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.533843 21741 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
18:48:19.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.539716 21776 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:48:19.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.541714 21741 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
18:48:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.542194 21741 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data,/tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "281d7f64b5fd4edeb2e8d3c6c469300b"
18:48:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:48:09 on dist-test-slave-9qt6"
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.545526 21741 fs_report.cc:383] FS layout report
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/data
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:48:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:48:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.611544 21741 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:48:19.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.620455 21741 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:48:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.621053 21741 kserver.cc:163] Server-wide thread pool size limit: 3276
18:48:19.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.710803 21741 rpc_server.cc:307] RPC server started. Bound to: 127.20.206.254:39439
18:48:19.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.710942 21827 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.206.254:39439 every 8 connection(s)
18:48:19.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.714898 21741 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/data/info.pb
18:48:19.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.723238 21307 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu as pid 21741
18:48:19.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:48:19.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.728896 21828 sys_catalog.cc:263] Verifying existing consensus state
18:48:19.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 18:48:19.729933 21828 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
18:48:19.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.730507 21741 master.cc:537] Master@127.20.206.254:39439 shutting down...
18:48:19.785 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:19.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:19.812896 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:19.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:19.813812 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:19.843737 21741 master.cc:552] Master@127.20.206.254:39439 shutdown complete.
18:48:19.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvcoJf5/test-tmp/mini-kudu-cluster708227465280487512/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
18:48:19.875 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.206.252:35459(127.20.206.252:35459)] connecting to peer
18:48:19.876 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:19.932 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.206.252:35459(127.20.206.252:35459)] Successfully connected to peer
18:48:19.935 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:19.935 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:19.937 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:19.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:19.939 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:19.943 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.048 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
18:48:20.065 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.206.252
18:48:20.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:20.200223 21834 raft_consensus.cc:482] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:48:20.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:20.200920 21834 raft_consensus.cc:504] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:20.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:20.204533 21834 leader_election.cc:288] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439), 93485ec69e05410b8daf427106313eea (127.20.206.253:44071)
18:48:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:20.213174 21690 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439): Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111)
18:48:20.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:20.213670 21689 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 93485ec69e05410b8daf427106313eea (127.20.206.253:44071): Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:20.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:20.214519 21689 leader_election.cc:302] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 09106035e7bf4087b858c3eb5f795743; no voters: 281d7f64b5fd4edeb2e8d3c6c469300b, 93485ec69e05410b8daf427106313eea
18:48:20.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:20.215533 21834 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:48:20.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:20 dist-test-slave-9qt6 krb5kdc[21320](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894894, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.206.252@KRBTEST.COM
18:48:20.393 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf6eacc9f, L:/127.0.0.1:37484 - R:/127.20.206.252:35459] using SASL/GSSAPI
18:48:20.410 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:20.431 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:20.437 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.438 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:20.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.440 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.448 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:20.470 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:20.474 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:20.476 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.477 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.482 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.483 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.484 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.484 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:20.491 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:20.492 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:20.495 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.495 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.496 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.503 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.504 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.505 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.507 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:20.530 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:20.533 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:20.533 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.536 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.540 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:20.550 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:20.552 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:20.553 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.554 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.555 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.557 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.557 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.559 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:20.610 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:20.612 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:20.615 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.615 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.617 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.620 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.622 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.623 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:20.690 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:20.693 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:20.695 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.695 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.695 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.697 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.697 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:20.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:20.818904 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:20.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:20.819285 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:20.890 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:20.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:20.893 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.893 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.894 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:20.895 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.896 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:20.898 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:21.070 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:21.072 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:21.074 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:21.074 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:21.074 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:21.075 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:21.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:21.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:21.081 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:21.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:21.822067 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:21.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:21.822365 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:22.070 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:22.075 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:22.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:22.075 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:22.075 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:22.077 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:22.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:22.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:22.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:22.330 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:22.332 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:22.334 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:22.334 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:22.335 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:22.335 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:22.335 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:22.335 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:22.337 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:22.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:22.825139 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:22.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:22.825410 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:22.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:22.838546 21668 master_runner.cc:186] Error getting master registration for 127.20.206.254:39439: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111),
18:48:22.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:22.838979 21668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:48:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:22.906407 21841 raft_consensus.cc:482] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:48:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:22.906857 21841 raft_consensus.cc:504] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:22.908463 21841 leader_election.cc:288] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439), 93485ec69e05410b8daf427106313eea (127.20.206.253:44071)
18:48:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:22.913463 21689 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 93485ec69e05410b8daf427106313eea (127.20.206.253:44071): Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:22.914034 21690 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439): Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111)
18:48:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:22.914515 21690 leader_election.cc:302] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 09106035e7bf4087b858c3eb5f795743; no voters: 281d7f64b5fd4edeb2e8d3c6c469300b, 93485ec69e05410b8daf427106313eea
18:48:22.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:22.915194 21841 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:48:23.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:23.828260 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:23.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:23.828521 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:24.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:24.831333 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:24.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:24.831588 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:25.690 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:25.691 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:25.694 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:25.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:25.695 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:25.695 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:25.696 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:25.696 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:25.697 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:25.834376 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:25.834632 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:26.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:26.705744 21850 raft_consensus.cc:482] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:48:26.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:26.706391 21850 raft_consensus.cc:504] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:26.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:26.708298 21850 leader_election.cc:288] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439), 93485ec69e05410b8daf427106313eea (127.20.206.253:44071)
18:48:26.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:26.710647 21690 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111) [suppressed 5 similar messages]
18:48:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:26.713804 21689 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 93485ec69e05410b8daf427106313eea (127.20.206.253:44071): Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:26.714382 21690 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439): Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111)
18:48:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:26.714695 21690 leader_election.cc:302] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 09106035e7bf4087b858c3eb5f795743; no voters: 281d7f64b5fd4edeb2e8d3c6c469300b, 93485ec69e05410b8daf427106313eea
18:48:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:26.715230 21850 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:48:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:26.837850 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:26.838141 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:26.950 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:26.951 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:26.953 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:26.954 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:26.954 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:26.954 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:26.954 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:26.954 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:26.956 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:27 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894907, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:27.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:27.507754 21683 init.cc:295] Successfully reacquired a new kerberos TGT
18:48:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:27.841198 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:27.841540 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:27.842505 21668 master_runner.cc:186] Error getting master registration for 127.20.206.254:39439: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111),
18:48:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:27.842871 21668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:48:28.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:28.343451 21152 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 21020
18:48:28.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:28.344440 21152 debug-util.cc:398] Leaking SignalData structure 0x7b08000af1e0 after lost signal to thread 21155
18:48:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:28.845548 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:28.845890 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:29.651 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:29.652 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:29.654 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:29.654 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:29.655 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:29.656 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:29.656 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:29.656 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:29.658 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:29.848551 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:29.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:29.848798 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:30.630 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:30.632 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:30.633 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:30.634 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:30.635 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:30.635 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:30.635 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:30.636 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:30.636 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:30.851519 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:30.851768 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:31.039417 21856 raft_consensus.cc:482] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:48:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:31.039809 21856 raft_consensus.cc:504] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:31.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:31.041605 21856 leader_election.cc:288] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439), 93485ec69e05410b8daf427106313eea (127.20.206.253:44071)
18:48:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:31.047362 21690 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439): Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111)
18:48:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:31.047850 21689 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 93485ec69e05410b8daf427106313eea (127.20.206.253:44071): Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:31.048204 21689 leader_election.cc:302] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 09106035e7bf4087b858c3eb5f795743; no voters: 281d7f64b5fd4edeb2e8d3c6c469300b, 93485ec69e05410b8daf427106313eea
18:48:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:31.048779 21856 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:48:31.171 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:31.172 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:31.174 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:31.174 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:31.175 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:31.175 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:31.175 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:31.175 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:31.176 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:31.770 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:31.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:31.774 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:31.774 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:31.775 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:31.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:31.777 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:31.777 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:31.777 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:31.854523 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:31.854765 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:32.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:32.845762 21668 master_runner.cc:186] Error getting master registration for 127.20.206.254:39439: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111),
18:48:32.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:32.846174 21668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:48:32.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:32.857884 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:32.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:32.858268 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:33.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:33.861455 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:33.861775 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:34.410 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:34.411 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:34.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:34.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:34.414 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:34.415 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:34.416 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:34.416 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:34.417 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:34.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:34.865142 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:34.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:34.865501 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:35.869519 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:35.869872 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:36.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:36.874048 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:36.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:36.874480 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:37.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:37.471 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:37.475 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:37.475 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:37.476 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:37.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:37.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:37.482 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:37.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:37 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894917, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:37.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:37.511150 21683 init.cc:295] Successfully reacquired a new kerberos TGT
18:48:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:37.760107 21871 raft_consensus.cc:482] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:48:37.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:37.760555 21871 raft_consensus.cc:504] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281d7f64b5fd4edeb2e8d3c6c469300b" member_type: VOTER last_known_addr { host: "127.20.206.254" port: 39439 } } peers { permanent_uuid: "93485ec69e05410b8daf427106313eea" member_type: VOTER last_known_addr { host: "127.20.206.253" port: 44071 } } peers { permanent_uuid: "09106035e7bf4087b858c3eb5f795743" member_type: VOTER last_known_addr { host: "127.20.206.252" port: 35459 } }
18:48:37.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:37.762213 21871 leader_election.cc:288] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439), 93485ec69e05410b8daf427106313eea (127.20.206.253:44071)
18:48:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:37.764856 21690 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111) [suppressed 3 similar messages]
18:48:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:37.768703 21689 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 93485ec69e05410b8daf427106313eea (127.20.206.253:44071): Network error: Client connection negotiation failed: client connection to 127.20.206.253:44071: connect: Connection refused (error 111)
18:48:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:37.769245 21690 leader_election.cc:334] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 281d7f64b5fd4edeb2e8d3c6c469300b (127.20.206.254:39439): Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111)
18:48:37.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:37.769718 21690 leader_election.cc:302] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 09106035e7bf4087b858c3eb5f795743; no voters: 281d7f64b5fd4edeb2e8d3c6c469300b, 93485ec69e05410b8daf427106313eea
18:48:37.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:37.770517 21871 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:48:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:37.849074 21668 master_runner.cc:186] Error getting master registration for 127.20.206.254:39439: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111),
18:48:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:37.849550 21668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:48:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:37.877568 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:37.877921 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:37.990 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:37.992 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:37.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:37.993 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:37.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:37.993 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:37.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:37.994 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:37.996 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:38.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:38.039619 20805 debug-util.cc:398] Leaking SignalData structure 0x7b080009e960 after lost signal to thread 20737
18:48:38.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:38.040447 20805 debug-util.cc:398] Leaking SignalData structure 0x7b080009e580 after lost signal to thread 20808
18:48:38.350 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:38.351 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:38.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:38.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:38.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:38.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:38.354 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:38.354 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:38.357 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:38.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:38.503948 20651 debug-util.cc:398] Leaking SignalData structure 0x7b08000c24c0 after lost signal to thread 20588
18:48:38.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:38.504840 20651 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5be0 after lost signal to thread 20654
18:48:38.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:38.881721 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:38.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:38.882045 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:39.885216 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:39.885527 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:40.888563 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:40.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:40.888877 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:41.350 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:41.351 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:41.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:41.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:41.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:41.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:41.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:41.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:41.355 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:41.892196 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:41.892509 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:42.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:42.852311 21668 master_runner.cc:186] Error getting master registration for 127.20.206.254:39439: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111),
18:48:42.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:42.852700 21668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:48:42.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:42.896224 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:42.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:42.896628 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:43.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:43.631793 21292 debug-util.cc:398] Leaking SignalData structure 0x7b08000af7a0 after lost signal to thread 21167
18:48:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:43.632786 21292 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff20 after lost signal to thread 21295
18:48:43.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:43.899845 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:43.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:43.900151 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:43.923164 20726 debug-util.cc:398] Leaking SignalData structure 0x7b0800090140 after lost signal to thread 20661
18:48:43.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:43.923895 20726 debug-util.cc:398] Leaking SignalData structure 0x7b0800093240 after lost signal to thread 20729
18:48:44.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:44.903685 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:44.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:44.904001 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:45.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:45.137390 21004 debug-util.cc:398] Leaking SignalData structure 0x7b08000aebc0 after lost signal to thread 20876
18:48:45.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:45.138372 21004 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8ce0 after lost signal to thread 21007
18:48:45.450 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:45.451 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:45.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:45.453 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:45.453 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:45.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:45.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:45.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:45.457 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:45.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:45.907166 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:45.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:45.907872 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:46.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:46.912165 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:46.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:46.912662 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:47.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:48:47 dist-test-slave-9qt6 krb5kdc[21320](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781894927, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.206.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:48:47.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:47.514093 21683 init.cc:295] Successfully reacquired a new kerberos TGT
18:48:47.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:47.856343 21668 master_runner.cc:186] Error getting master registration for 127.20.206.254:39439: Network error: Client connection negotiation failed: client connection to 127.20.206.254:39439: connect: Connection refused (error 111),
18:48:47.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:47.856838 21668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:48:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:47.915474 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:47.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:47.915743 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:48.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:48.918322 21754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 09106035e7bf4087b858c3eb5f795743: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:48:48.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:48.918606 21754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:48:48.950 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.253:44071(127.20.206.253:44071)] connecting to peer
18:48:48.951 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.206.254:39439(127.20.206.254:39439)] connecting to peer
18:48:48.953 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:48.953 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:48.953 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.253:44071(127.20.206.253:44071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:48.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.206.254:39439(127.20.206.254:39439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:48.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.253:44071: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071
18:48:48.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.206.254:39439: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
18:48:48.956 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439; will retry
18:48:48.962 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29228), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [205ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [677ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [704ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [707ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [714ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [715ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [747ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [749ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [750ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [761ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [769ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [773ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [802ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [804ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [806ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [820ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [823ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [825ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [882ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [885ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [887ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [889ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [964ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1155ms] refreshing cache from master, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1336ms] refreshing cache from master, [1336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1346ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1347ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2345ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2595ms] refreshing cache from master, [2595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2600ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2601ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2602ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2603ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [5955ms] refreshing cache from master, [5956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [5961ms] too many traces: truncated at 100 traces, deferred=Deferred@772451740(state=PENDING, result=null, callback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801), errback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801)))
18:48:48.967 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.206.253:44071(127.20.206.253:44071)] disconnecting while in state TERMINATED
18:48:48.968 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.206.254:39439(127.20.206.254:39439)] disconnecting while in state TERMINATED
18:48:48.968 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.206.252:35459(127.20.206.252:35459)] disconnecting while in state READY
18:48:48.973 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:48:48.976 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.206.252:35459(127.20.206.252:35459)] handling channelInactive
18:48:48.976 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:48:48.976 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.206.252:35459(127.20.206.252:35459)] cleaning up while in state READY due to: connection closed
18:48:48.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:48.980392 21307 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 21668
18:48:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:48:50Z chronyd exiting
18:48:50.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:50.152977 21152 debug-util.cc:398] Leaking SignalData structure 0x7b08000af9e0 after lost signal to thread 21020
18:48:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:48:50.153919 21152 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbc00 after lost signal to thread 21155
18:48:51.144 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:48:51.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:51.145743 20575 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 20856
18:48:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:51.172683 20575 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 21017
18:48:51.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:51.199033 20575 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 21163
18:48:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:51.226919 20575 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 20587
18:48:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:51.256318 20575 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 20657
18:48:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:48:51.291083 20575 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvcoJf5/build/tsan/bin/kudu with pid 20732
18:48:52.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:48:52Z chronyd exiting
18:48:53.412 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29228), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [205ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [677ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [704ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [707ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [714ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [715ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [747ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [749ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [750ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [761ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [769ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [773ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [802ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [804ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [806ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [820ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [823ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [825ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [882ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [885ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [887ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [889ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [964ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1155ms] refreshing cache from master, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1336ms] refreshing cache from master, [1336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1346ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1347ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2345ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2595ms] refreshing cache from master, [2595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2600ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2601ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2602ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2603ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [5955ms] refreshing cache from master, [5956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [5961ms] too many traces: truncated at 100 traces, deferred=Deferred@772451740(state=PENDING, result=null, callback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801), errback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.206.254:39439
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29228), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [205ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [677ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [704ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [707ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [714ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [715ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [747ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [749ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [750ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [761ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [769ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [773ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [802ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [804ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [806ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [820ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [823ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [825ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [882ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [885ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [887ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [889ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [964ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1155ms] refreshing cache from master, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1336ms] refreshing cache from master, [1336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1346ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1347ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2345ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2595ms] refreshing cache from master, [2595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2600ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2601ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2602ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2603ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [5955ms] refreshing cache from master, [5956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [5961ms] too many traces: truncated at 100 traces, deferred=Deferred@772451740(state=PENDING, result=null, callback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801), errback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.206.254:39439
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
18:48:54.516 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
18:48:54.516 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29228), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [205ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [677ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [704ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [707ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [714ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [715ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [747ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [749ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [750ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [761ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [769ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [773ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [802ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [804ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [806ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [820ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [823ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [825ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [882ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [885ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [887ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [889ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [964ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1155ms] refreshing cache from master, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1336ms] refreshing cache from master, [1336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1346ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1347ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2345ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2595ms] refreshing cache from master, [2595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2600ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2601ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2602ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2603ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [5955ms] refreshing cache from master, [5956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [5961ms] too many traces: truncated at 100 traces, deferred=Deferred@772451740(state=PENDING, result=null, callback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801), errback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.206.254:39439
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29228), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [205ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [677ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [704ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [707ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [714ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [715ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [747ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [749ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [750ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [761ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [769ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [773ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [802ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [804ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [806ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [820ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [823ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [825ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [882ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [885ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [887ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [889ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [961ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [964ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1155ms] refreshing cache from master, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1336ms] refreshing cache from master, [1336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [1346ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [1347ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2341ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2344ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2345ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2595ms] refreshing cache from master, [2595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [2596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [2597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [2600ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.254:39439: Network error: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439, [2601ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.253:44071: Network error: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [2602ms] Sub RPC ConnectToMaster: received response from server master-127.20.206.252:35459: OK, [2603ms] delaying RPC due to: Service unavailable: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071, [5955ms] refreshing cache from master, [5956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.253:44071, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.252:35459, [5957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.206.254:39439, [5961ms] too many traces: truncated at 100 traces, deferred=Deferred@772451740(state=PENDING, result=null, callback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801), errback=(continuation of Deferred@1257082533 after retry RPC after error@588744645) -> (continuation of Deferred@1010596448 after retry RPC after error@1201755702) -> (continuation of Deferred@1637839041 after retry RPC after error@2116680344) -> (continuation of Deferred@555315271 after retry RPC after error@1824817844) -> (continuation of Deferred@1984429717 after retry RPC after error@1191917567) -> (continuation of Deferred@763034641 after retry RPC after error@1700989480) -> (continuation of Deferred@748286791 after retry RPC after error@850632164) -> (continuation of Deferred@1221817815 after retry RPC after error@891032751) -> (continuation of Deferred@1451022976 after retry RPC after error@2022584598) -> (continuation of Deferred@731946525 after retry RPC after error@2127322794) -> (continuation of Deferred@1322525481 after retry RPC after error@1197194596) -> (continuation of Deferred@1755616228 after retry RPC after error@1954658644) -> (continuation of Deferred@455278982 after retry RPC after error@17383195) -> (continuation of Deferred@1308538733 after retry RPC after error@877460639) -> (continuation of Deferred@937485473 after retry RPC after error@2136597665) -> (continuation of Deferred@1193566712 after retry RPC after error@350213511) -> (continuation of Deferred@615181905 after retry RPC after error@350648969) -> (continuation of Deferred@1881797196 after retry RPC after error@439382563) -> (continuation of Deferred@1771822930 after retry RPC after error@1317978842) -> (continuation of Deferred@1904931541 after retry RPC after error@1748121517) -> (continuation of Deferred@1600394422 after retry RPC after error@1732547986) -> (continuation of Deferred@1098829813 after retry RPC after error@940050216) -> (continuation of Deferred@1203135803 after retry RPC after error@77430874) -> (continuation of Deferred@866907739 after retry RPC after error@1273140801)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.206.253:44071,127.20.206.252:35459,127.20.206.254:39439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.253:44071(127.20.206.253:44071): Connection refused: /127.20.206.253:44071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.206.254:39439(127.20.206.254:39439): Connection refused: /127.20.206.254:39439
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.206.254:39439
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)